2024-11-17 21:23:56,933 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 21:23:56,953 main DEBUG Took 0.010029 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-17 21:23:56,953 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-17 21:23:56,953 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-17 21:23:56,954 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-17 21:23:56,955 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 21:23:56,964 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-17 21:23:56,977 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 21:23:56,979 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 21:23:56,980 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 21:23:56,981 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 21:23:56,981 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 21:23:56,982 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 21:23:56,983 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 21:23:56,984 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 21:23:56,984 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 21:23:56,985 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 21:23:56,986 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 21:23:56,987 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 21:23:56,987 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 21:23:56,988 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 21:23:56,988 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 21:23:56,989 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 21:23:56,990 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 21:23:56,990 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 21:23:56,991 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 21:23:56,991 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 21:23:56,992 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 21:23:56,992 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 21:23:56,993 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 21:23:56,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 21:23:56,994 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 21:23:56,995 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-17 21:23:56,997 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 21:23:56,999 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-17 21:23:57,001 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-17 21:23:57,002 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-17 21:23:57,004 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-17 21:23:57,004 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-17 21:23:57,015 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-17 21:23:57,018 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-17 21:23:57,021 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-17 21:23:57,021 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-17 21:23:57,022 main DEBUG createAppenders(={Console}) 2024-11-17 21:23:57,023 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-17 21:23:57,024 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 21:23:57,025 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-17 21:23:57,025 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-17 21:23:57,026 main DEBUG OutputStream closed 2024-11-17 21:23:57,026 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-17 21:23:57,027 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-17 21:23:57,027 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-17 21:23:57,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-17 21:23:57,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-17 21:23:57,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-17 21:23:57,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-17 21:23:57,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-17 21:23:57,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-17 21:23:57,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-17 21:23:57,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-17 21:23:57,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-17 21:23:57,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-17 21:23:57,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-17 21:23:57,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-17 21:23:57,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-17 21:23:57,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-17 21:23:57,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-17 21:23:57,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-17 21:23:57,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-17 21:23:57,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-17 21:23:57,111 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17 21:23:57,111 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-17 21:23:57,111 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-17 21:23:57,112 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-17T21:23:57,359 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76 2024-11-17 21:23:57,363 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-17 21:23:57,364 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17T21:23:57,376 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-17T21:23:57,406 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-17T21:23:57,444 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/cluster_6acd51c4-a1e3-75e6-757b-0efaa889834a, deleteOnExit=true 2024-11-17T21:23:57,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-17T21:23:57,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/test.cache.data in system properties and HBase conf 2024-11-17T21:23:57,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/hadoop.tmp.dir in system properties and HBase conf 2024-11-17T21:23:57,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/hadoop.log.dir in system properties and HBase conf 2024-11-17T21:23:57,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-17T21:23:57,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-17T21:23:57,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-17T21:23:57,585 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-17T21:23:57,693 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T21:23:57,698 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-17T21:23:57,699 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-17T21:23:57,699 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-17T21:23:57,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T21:23:57,701 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-17T21:23:57,701 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-17T21:23:57,702 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T21:23:57,702 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T21:23:57,703 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-17T21:23:57,703 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/nfs.dump.dir in system properties and HBase conf 2024-11-17T21:23:57,703 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/java.io.tmpdir in system properties and HBase conf 2024-11-17T21:23:57,704 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T21:23:57,704 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-17T21:23:57,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-17T21:23:58,801 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-17T21:23:58,868 INFO [Time-limited test {}] log.Log(170): Logging initialized @2816ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-17T21:23:58,931 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T21:23:58,986 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T21:23:59,010 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T21:23:59,010 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T21:23:59,011 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-17T21:23:59,022 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T21:23:59,025 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/hadoop.log.dir/,AVAILABLE} 2024-11-17T21:23:59,026 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T21:23:59,198 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@322e478f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/java.io.tmpdir/jetty-localhost-42975-hadoop-hdfs-3_4_1-tests_jar-_-any-12524527224765490873/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T21:23:59,210 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:42975} 2024-11-17T21:23:59,210 INFO [Time-limited test {}] server.Server(415): Started @3159ms 2024-11-17T21:23:59,750 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T21:23:59,757 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T21:23:59,758 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T21:23:59,759 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T21:23:59,759 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-17T21:23:59,759 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/hadoop.log.dir/,AVAILABLE} 2024-11-17T21:23:59,760 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T21:23:59,859 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@42a79998{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/java.io.tmpdir/jetty-localhost-36815-hadoop-hdfs-3_4_1-tests_jar-_-any-4323821416432471265/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T21:23:59,859 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:36815} 2024-11-17T21:23:59,860 INFO [Time-limited test {}] server.Server(415): Started @3809ms 2024-11-17T21:23:59,908 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T21:24:01,028 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/cluster_6acd51c4-a1e3-75e6-757b-0efaa889834a/data/data2/current/BP-2092338727-172.17.0.2-1731878638249/current, will proceed with Du for space computation calculation, 2024-11-17T21:24:01,028 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/cluster_6acd51c4-a1e3-75e6-757b-0efaa889834a/data/data1/current/BP-2092338727-172.17.0.2-1731878638249/current, will proceed with Du for space computation calculation, 2024-11-17T21:24:01,053 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T21:24:01,093 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76 2024-11-17T21:24:01,096 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8b702fa711e96c26 with lease ID 0x31a22af92472f6e0: Processing first storage report for DS-df620682-9f57-47c1-b861-3d6acafd468f from datanode DatanodeRegistration(127.0.0.1:35609, datanodeUuid=f9e820aa-39dd-40f9-8116-567c831aafd3, infoPort=43631, infoSecurePort=0, ipcPort=40505, storageInfo=lv=-57;cid=testClusterID;nsid=2136367785;c=1731878638249) 2024-11-17T21:24:01,097 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8b702fa711e96c26 with lease ID 0x31a22af92472f6e0: from storage DS-df620682-9f57-47c1-b861-3d6acafd468f node DatanodeRegistration(127.0.0.1:35609, datanodeUuid=f9e820aa-39dd-40f9-8116-567c831aafd3, infoPort=43631, infoSecurePort=0, ipcPort=40505, storageInfo=lv=-57;cid=testClusterID;nsid=2136367785;c=1731878638249), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-17T21:24:01,098 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8b702fa711e96c26 with lease ID 0x31a22af92472f6e0: Processing first storage report for DS-aa92b38b-4773-4ddd-8bbd-d7b54a1e0de1 from datanode DatanodeRegistration(127.0.0.1:35609, datanodeUuid=f9e820aa-39dd-40f9-8116-567c831aafd3, infoPort=43631, infoSecurePort=0, ipcPort=40505, storageInfo=lv=-57;cid=testClusterID;nsid=2136367785;c=1731878638249) 2024-11-17T21:24:01,098 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8b702fa711e96c26 with lease ID 0x31a22af92472f6e0: from storage DS-aa92b38b-4773-4ddd-8bbd-d7b54a1e0de1 node DatanodeRegistration(127.0.0.1:35609, datanodeUuid=f9e820aa-39dd-40f9-8116-567c831aafd3, infoPort=43631, infoSecurePort=0, ipcPort=40505, storageInfo=lv=-57;cid=testClusterID;nsid=2136367785;c=1731878638249), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T21:24:01,166 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/cluster_6acd51c4-a1e3-75e6-757b-0efaa889834a/zookeeper_0, clientPort=62975, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/cluster_6acd51c4-a1e3-75e6-757b-0efaa889834a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/cluster_6acd51c4-a1e3-75e6-757b-0efaa889834a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-17T21:24:01,175 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62975 2024-11-17T21:24:01,183 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T21:24:01,186 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T21:24:01,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741825_1001 (size=7) 2024-11-17T21:24:01,809 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249 with version=8 2024-11-17T21:24:01,809 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/hbase-staging 2024-11-17T21:24:01,900 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-17T21:24:02,100 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a313eea8709e:0 server-side Connection retries=45 2024-11-17T21:24:02,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T21:24:02,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T21:24:02,114 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T21:24:02,114 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T21:24:02,114 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T21:24:02,251 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-17T21:24:02,304 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-17T21:24:02,316 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-17T21:24:02,319 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T21:24:02,342 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 33075 (auto-detected) 2024-11-17T21:24:02,343 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-17T21:24:02,362 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43623 2024-11-17T21:24:02,385 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43623 connecting to ZooKeeper ensemble=127.0.0.1:62975 2024-11-17T21:24:02,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:436230x0, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T21:24:02,569 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43623-0x1014aaed9aa0000 connected 2024-11-17T21:24:02,664 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T21:24:02,666 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T21:24:02,675 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T21:24:02,680 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249, hbase.cluster.distributed=false 2024-11-17T21:24:02,711 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T21:24:02,718 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43623 2024-11-17T21:24:02,719 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43623 2024-11-17T21:24:02,720 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43623 2024-11-17T21:24:02,728 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43623 2024-11-17T21:24:02,728 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43623 2024-11-17T21:24:02,874 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a313eea8709e:0 server-side Connection retries=45 2024-11-17T21:24:02,876 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T21:24:02,877 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T21:24:02,877 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T21:24:02,877 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T21:24:02,877 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T21:24:02,881 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T21:24:02,886 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T21:24:02,888 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46445 2024-11-17T21:24:02,891 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46445 connecting to ZooKeeper ensemble=127.0.0.1:62975 2024-11-17T21:24:02,896 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T21:24:02,900 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T21:24:02,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:464450x0, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T21:24:02,921 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:464450x0, quorum=127.0.0.1:62975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T21:24:02,927 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T21:24:02,927 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46445-0x1014aaed9aa0001 connected 2024-11-17T21:24:02,939 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T21:24:02,943 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T21:24:02,953 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T21:24:02,954 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46445 2024-11-17T21:24:02,956 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46445 2024-11-17T21:24:02,960 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46445 2024-11-17T21:24:02,965 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46445 2024-11-17T21:24:02,966 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46445 2024-11-17T21:24:02,995 DEBUG [M:0;a313eea8709e:43623 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a313eea8709e:43623 2024-11-17T21:24:02,996 INFO [master/a313eea8709e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a313eea8709e,43623,1731878641945 2024-11-17T21:24:03,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T21:24:03,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T21:24:03,012 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a313eea8709e,43623,1731878641945 2024-11-17T21:24:03,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T21:24:03,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T21:24:03,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T21:24:03,044 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T21:24:03,046 INFO [master/a313eea8709e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a313eea8709e,43623,1731878641945 from backup master directory 2024-11-17T21:24:03,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T21:24:03,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a313eea8709e,43623,1731878641945 2024-11-17T21:24:03,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T21:24:03,060 WARN [master/a313eea8709e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T21:24:03,060 INFO [master/a313eea8709e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a313eea8709e,43623,1731878641945 2024-11-17T21:24:03,063 INFO [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-17T21:24:03,065 INFO [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-17T21:24:03,129 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/hbase.id] with ID: 8f8173f4-7420-463b-8a7b-f3fcd9daf410 2024-11-17T21:24:03,130 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/.tmp/hbase.id 2024-11-17T21:24:03,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741826_1002 (size=42) 2024-11-17T21:24:03,545 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/.tmp/hbase.id]:[hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/hbase.id] 2024-11-17T21:24:03,603 INFO [master/a313eea8709e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T21:24:03,610 INFO [master/a313eea8709e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-17T21:24:03,636 INFO [master/a313eea8709e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-17T21:24:03,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T21:24:03,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T21:24:03,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741827_1003 (size=196) 2024-11-17T21:24:04,114 INFO [master/a313eea8709e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T21:24:04,117 INFO [master/a313eea8709e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-17T21:24:04,136 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:04,142 INFO [master/a313eea8709e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T21:24:04,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741828_1004 (size=1189) 2024-11-17T21:24:04,597 INFO [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store 2024-11-17T21:24:04,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741829_1005 (size=34) 2024-11-17T21:24:05,019 INFO [master/a313eea8709e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-17T21:24:05,023 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:24:05,024 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T21:24:05,024 INFO [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T21:24:05,025 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T21:24:05,027 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T21:24:05,027 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T21:24:05,027 INFO [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T21:24:05,029 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731878645024Disabling compacts and flushes for region at 1731878645024Disabling writes for close at 1731878645027 (+3 ms)Writing region close event to WAL at 1731878645027Closed at 1731878645027 2024-11-17T21:24:05,032 WARN [master/a313eea8709e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/.initializing 2024-11-17T21:24:05,032 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/WALs/a313eea8709e,43623,1731878641945 2024-11-17T21:24:05,042 INFO [master/a313eea8709e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T21:24:05,059 INFO [master/a313eea8709e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a313eea8709e%2C43623%2C1731878641945, suffix=, logDir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/WALs/a313eea8709e,43623,1731878641945, archiveDir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/oldWALs, maxLogs=10 2024-11-17T21:24:05,090 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/WALs/a313eea8709e,43623,1731878641945/a313eea8709e%2C43623%2C1731878641945.1731878645065, exclude list is [], retry=0 2024-11-17T21:24:05,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35609,DS-df620682-9f57-47c1-b861-3d6acafd468f,DISK] 2024-11-17T21:24:05,113 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-17T21:24:05,160 INFO [master/a313eea8709e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/WALs/a313eea8709e,43623,1731878641945/a313eea8709e%2C43623%2C1731878641945.1731878645065 2024-11-17T21:24:05,162 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43631:43631)] 2024-11-17T21:24:05,162 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-17T21:24:05,163 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:24:05,167 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T21:24:05,168 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T21:24:05,201 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T21:24:05,222 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-17T21:24:05,226 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:05,228 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T21:24:05,228 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T21:24:05,232 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-17T21:24:05,232 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:05,233 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:24:05,233 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T21:24:05,236 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-17T21:24:05,236 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:05,238 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:24:05,238 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T21:24:05,240 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-17T21:24:05,241 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:05,242 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:24:05,242 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T21:24:05,246 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T21:24:05,248 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T21:24:05,253 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T21:24:05,253 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T21:24:05,257 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T21:24:05,261 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T21:24:05,265 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T21:24:05,266 INFO [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61779046, jitterRate=-0.07942047715187073}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T21:24:05,273 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731878645179Initializing all the Stores at 1731878645181 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731878645181Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878645182 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878645182Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878645182Cleaning up temporary data from old regions at 1731878645253 (+71 ms)Region opened successfully at 1731878645273 (+20 ms) 2024-11-17T21:24:05,275 INFO [master/a313eea8709e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-17T21:24:05,309 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77168441, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a313eea8709e/172.17.0.2:0 2024-11-17T21:24:05,336 INFO [master/a313eea8709e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-17T21:24:05,345 INFO [master/a313eea8709e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-17T21:24:05,345 INFO [master/a313eea8709e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-17T21:24:05,347 INFO [master/a313eea8709e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T21:24:05,348 INFO [master/a313eea8709e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-17T21:24:05,352 INFO [master/a313eea8709e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-17T21:24:05,353 INFO [master/a313eea8709e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-17T21:24:05,375 INFO [master/a313eea8709e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-17T21:24:05,383 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-17T21:24:05,409 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-17T21:24:05,411 INFO [master/a313eea8709e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-17T21:24:05,413 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-17T21:24:05,426 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-17T21:24:05,429 INFO [master/a313eea8709e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-17T21:24:05,433 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-17T21:24:05,440 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-17T21:24:05,443 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-17T21:24:05,451 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-17T21:24:05,472 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-17T21:24:05,482 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-17T21:24:05,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T21:24:05,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T21:24:05,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T21:24:05,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T21:24:05,496 INFO [master/a313eea8709e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a313eea8709e,43623,1731878641945, sessionid=0x1014aaed9aa0000, setting cluster-up flag (Was=false) 2024-11-17T21:24:05,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T21:24:05,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T21:24:05,556 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-17T21:24:05,559 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a313eea8709e,43623,1731878641945 2024-11-17T21:24:05,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T21:24:05,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T21:24:05,615 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-17T21:24:05,617 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a313eea8709e,43623,1731878641945 2024-11-17T21:24:05,623 INFO [master/a313eea8709e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-17T21:24:05,677 INFO [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(746): ClusterId : 8f8173f4-7420-463b-8a7b-f3fcd9daf410 2024-11-17T21:24:05,680 DEBUG [RS:0;a313eea8709e:46445 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T21:24:05,695 DEBUG [RS:0;a313eea8709e:46445 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T21:24:05,695 DEBUG [RS:0;a313eea8709e:46445 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T21:24:05,705 DEBUG [RS:0;a313eea8709e:46445 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T21:24:05,705 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T21:24:05,706 DEBUG [RS:0;a313eea8709e:46445 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@752f493a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a313eea8709e/172.17.0.2:0 2024-11-17T21:24:05,716 INFO [master/a313eea8709e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-17T21:24:05,719 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a313eea8709e:46445 2024-11-17T21:24:05,722 INFO [RS:0;a313eea8709e:46445 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T21:24:05,723 INFO [RS:0;a313eea8709e:46445 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T21:24:05,723 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T21:24:05,724 INFO [master/a313eea8709e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-17T21:24:05,726 INFO [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(2659): reportForDuty to master=a313eea8709e,43623,1731878641945 with port=46445, startcode=1731878642828 2024-11-17T21:24:05,731 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a313eea8709e,43623,1731878641945 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-17T21:24:05,738 DEBUG [RS:0;a313eea8709e:46445 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T21:24:05,739 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a313eea8709e:0, corePoolSize=5, maxPoolSize=5 2024-11-17T21:24:05,740 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a313eea8709e:0, corePoolSize=5, maxPoolSize=5 2024-11-17T21:24:05,740 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a313eea8709e:0, corePoolSize=5, maxPoolSize=5 2024-11-17T21:24:05,740 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a313eea8709e:0, corePoolSize=5, maxPoolSize=5 2024-11-17T21:24:05,740 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a313eea8709e:0, corePoolSize=10, maxPoolSize=10 2024-11-17T21:24:05,741 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a313eea8709e:0, corePoolSize=1, maxPoolSize=1 2024-11-17T21:24:05,741 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a313eea8709e:0, corePoolSize=2, maxPoolSize=2 2024-11-17T21:24:05,741 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a313eea8709e:0, corePoolSize=1, maxPoolSize=1 2024-11-17T21:24:05,747 INFO [master/a313eea8709e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731878675746 2024-11-17T21:24:05,749 INFO [master/a313eea8709e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-17T21:24:05,750 INFO [master/a313eea8709e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-17T21:24:05,751 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T21:24:05,752 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-17T21:24:05,755 INFO [master/a313eea8709e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-17T21:24:05,755 INFO [master/a313eea8709e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-17T21:24:05,756 INFO [master/a313eea8709e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-17T21:24:05,756 INFO [master/a313eea8709e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-17T21:24:05,759 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:05,759 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-17T21:24:05,762 INFO [master/a313eea8709e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:05,771 INFO [master/a313eea8709e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-17T21:24:05,773 INFO [master/a313eea8709e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-17T21:24:05,773 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39297, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T21:24:05,774 INFO [master/a313eea8709e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-17T21:24:05,776 INFO [master/a313eea8709e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-17T21:24:05,776 INFO [master/a313eea8709e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-17T21:24:05,779 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a313eea8709e:0:becomeActiveMaster-HFileCleaner.large.0-1731878645778,5,FailOnTimeoutGroup] 2024-11-17T21:24:05,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741831_1007 (size=1321) 2024-11-17T21:24:05,781 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43623 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:05,784 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a313eea8709e:0:becomeActiveMaster-HFileCleaner.small.0-1731878645779,5,FailOnTimeoutGroup] 2024-11-17T21:24:05,784 INFO [master/a313eea8709e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:05,785 INFO [master/a313eea8709e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-17T21:24:05,786 INFO [master/a313eea8709e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:05,787 INFO [master/a313eea8709e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:05,806 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-17T21:24:05,806 WARN [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-17T21:24:05,907 INFO [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(2659): reportForDuty to master=a313eea8709e,43623,1731878641945 with port=46445, startcode=1731878642828 2024-11-17T21:24:05,909 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43623 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a313eea8709e,46445,1731878642828 2024-11-17T21:24:05,912 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43623 {}] master.ServerManager(517): Registering regionserver=a313eea8709e,46445,1731878642828 2024-11-17T21:24:05,919 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249 2024-11-17T21:24:05,920 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45699 2024-11-17T21:24:05,920 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T21:24:05,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T21:24:05,932 DEBUG [RS:0;a313eea8709e:46445 {}] zookeeper.ZKUtil(111): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a313eea8709e,46445,1731878642828 2024-11-17T21:24:05,932 WARN [RS:0;a313eea8709e:46445 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T21:24:05,932 INFO [RS:0;a313eea8709e:46445 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T21:24:05,932 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/WALs/a313eea8709e,46445,1731878642828 2024-11-17T21:24:05,935 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a313eea8709e,46445,1731878642828] 2024-11-17T21:24:05,958 INFO [RS:0;a313eea8709e:46445 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T21:24:05,975 INFO [RS:0;a313eea8709e:46445 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T21:24:05,981 INFO [RS:0;a313eea8709e:46445 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T21:24:05,981 INFO [RS:0;a313eea8709e:46445 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:05,983 INFO [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T21:24:05,988 INFO [RS:0;a313eea8709e:46445 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T21:24:05,990 INFO [RS:0;a313eea8709e:46445 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:05,991 DEBUG [RS:0;a313eea8709e:46445 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a313eea8709e:0, corePoolSize=1, maxPoolSize=1 2024-11-17T21:24:05,991 DEBUG [RS:0;a313eea8709e:46445 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a313eea8709e:0, corePoolSize=1, maxPoolSize=1 2024-11-17T21:24:05,991 DEBUG [RS:0;a313eea8709e:46445 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a313eea8709e:0, corePoolSize=1, maxPoolSize=1 2024-11-17T21:24:05,992 DEBUG [RS:0;a313eea8709e:46445 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a313eea8709e:0, corePoolSize=1, maxPoolSize=1 2024-11-17T21:24:05,992 DEBUG [RS:0;a313eea8709e:46445 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a313eea8709e:0, corePoolSize=1, maxPoolSize=1 2024-11-17T21:24:05,992 DEBUG [RS:0;a313eea8709e:46445 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a313eea8709e:0, corePoolSize=2, maxPoolSize=2 2024-11-17T21:24:05,992 DEBUG [RS:0;a313eea8709e:46445 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a313eea8709e:0, corePoolSize=1, maxPoolSize=1 2024-11-17T21:24:05,992 DEBUG [RS:0;a313eea8709e:46445 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a313eea8709e:0, corePoolSize=1, maxPoolSize=1 2024-11-17T21:24:05,993 DEBUG [RS:0;a313eea8709e:46445 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a313eea8709e:0, corePoolSize=1, maxPoolSize=1 2024-11-17T21:24:05,993 DEBUG [RS:0;a313eea8709e:46445 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a313eea8709e:0, corePoolSize=1, maxPoolSize=1 2024-11-17T21:24:05,993 DEBUG [RS:0;a313eea8709e:46445 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a313eea8709e:0, corePoolSize=1, maxPoolSize=1 2024-11-17T21:24:05,993 DEBUG [RS:0;a313eea8709e:46445 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a313eea8709e:0, corePoolSize=1, maxPoolSize=1 2024-11-17T21:24:05,993 DEBUG [RS:0;a313eea8709e:46445 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a313eea8709e:0, corePoolSize=3, maxPoolSize=3 2024-11-17T21:24:05,994 DEBUG [RS:0;a313eea8709e:46445 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0, corePoolSize=3, maxPoolSize=3 2024-11-17T21:24:05,995 INFO [RS:0;a313eea8709e:46445 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:05,996 INFO [RS:0;a313eea8709e:46445 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:05,996 INFO [RS:0;a313eea8709e:46445 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:05,996 INFO [RS:0;a313eea8709e:46445 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:05,996 INFO [RS:0;a313eea8709e:46445 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:05,996 INFO [RS:0;a313eea8709e:46445 {}] hbase.ChoreService(168): Chore ScheduledChore name=a313eea8709e,46445,1731878642828-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T21:24:06,025 INFO [RS:0;a313eea8709e:46445 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T21:24:06,027 INFO [RS:0;a313eea8709e:46445 {}] hbase.ChoreService(168): Chore ScheduledChore name=a313eea8709e,46445,1731878642828-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:06,028 INFO [RS:0;a313eea8709e:46445 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:06,028 INFO [RS:0;a313eea8709e:46445 {}] regionserver.Replication(171): a313eea8709e,46445,1731878642828 started 2024-11-17T21:24:06,047 INFO [RS:0;a313eea8709e:46445 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:06,047 INFO [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1482): Serving as a313eea8709e,46445,1731878642828, RpcServer on a313eea8709e/172.17.0.2:46445, sessionid=0x1014aaed9aa0001 2024-11-17T21:24:06,048 DEBUG [RS:0;a313eea8709e:46445 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T21:24:06,048 DEBUG [RS:0;a313eea8709e:46445 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a313eea8709e,46445,1731878642828 2024-11-17T21:24:06,048 DEBUG [RS:0;a313eea8709e:46445 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a313eea8709e,46445,1731878642828' 2024-11-17T21:24:06,048 DEBUG [RS:0;a313eea8709e:46445 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T21:24:06,049 DEBUG [RS:0;a313eea8709e:46445 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T21:24:06,050 DEBUG [RS:0;a313eea8709e:46445 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T21:24:06,050 DEBUG [RS:0;a313eea8709e:46445 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T21:24:06,050 DEBUG [RS:0;a313eea8709e:46445 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a313eea8709e,46445,1731878642828 2024-11-17T21:24:06,050 DEBUG [RS:0;a313eea8709e:46445 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a313eea8709e,46445,1731878642828' 2024-11-17T21:24:06,051 DEBUG [RS:0;a313eea8709e:46445 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T21:24:06,051 DEBUG [RS:0;a313eea8709e:46445 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T21:24:06,052 DEBUG [RS:0;a313eea8709e:46445 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T21:24:06,052 INFO [RS:0;a313eea8709e:46445 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T21:24:06,052 INFO [RS:0;a313eea8709e:46445 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T21:24:06,157 INFO [RS:0;a313eea8709e:46445 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T21:24:06,160 INFO [RS:0;a313eea8709e:46445 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a313eea8709e%2C46445%2C1731878642828, suffix=, logDir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/WALs/a313eea8709e,46445,1731878642828, archiveDir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/oldWALs, maxLogs=32 2024-11-17T21:24:06,176 DEBUG [RS:0;a313eea8709e:46445 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/WALs/a313eea8709e,46445,1731878642828/a313eea8709e%2C46445%2C1731878642828.1731878646163, exclude list is [], retry=0 2024-11-17T21:24:06,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35609,DS-df620682-9f57-47c1-b861-3d6acafd468f,DISK] 2024-11-17T21:24:06,184 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-17T21:24:06,184 INFO [RS:0;a313eea8709e:46445 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/WALs/a313eea8709e,46445,1731878642828/a313eea8709e%2C46445%2C1731878642828.1731878646163 2024-11-17T21:24:06,184 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249 2024-11-17T21:24:06,185 DEBUG [RS:0;a313eea8709e:46445 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43631:43631)] 2024-11-17T21:24:06,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741833_1009 (size=32) 2024-11-17T21:24:06,600 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:24:06,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T21:24:06,609 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T21:24:06,609 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:06,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T21:24:06,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T21:24:06,614 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T21:24:06,614 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:06,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T21:24:06,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T21:24:06,617 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T21:24:06,617 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:06,618 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T21:24:06,619 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T21:24:06,621 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T21:24:06,621 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:06,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T21:24:06,622 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T21:24:06,623 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740 2024-11-17T21:24:06,624 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740 2024-11-17T21:24:06,627 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T21:24:06,627 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T21:24:06,628 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T21:24:06,630 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T21:24:06,634 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T21:24:06,634 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63323448, jitterRate=-0.05640709400177002}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T21:24:06,637 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731878646600Initializing all the Stores at 1731878646604 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731878646604Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731878646604Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878646605 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731878646605Cleaning up temporary data from old regions at 1731878646627 (+22 ms)Region opened successfully at 1731878646637 (+10 ms) 2024-11-17T21:24:06,637 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T21:24:06,637 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T21:24:06,637 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T21:24:06,637 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T21:24:06,637 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T21:24:06,638 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T21:24:06,639 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731878646637Disabling compacts and flushes for region at 1731878646637Disabling writes for close at 1731878646637Writing region close event to WAL at 1731878646638 (+1 ms)Closed at 1731878646638 2024-11-17T21:24:06,642 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T21:24:06,642 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-17T21:24:06,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T21:24:06,654 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T21:24:06,657 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-17T21:24:06,809 DEBUG [a313eea8709e:43623 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-17T21:24:06,819 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:24:06,825 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a313eea8709e,46445,1731878642828, state=OPENING 2024-11-17T21:24:06,875 DEBUG [PEWorker-5 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-17T21:24:06,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T21:24:06,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T21:24:06,928 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T21:24:06,928 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T21:24:06,931 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T21:24:06,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:24:07,114 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T21:24:07,117 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35683, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T21:24:07,127 INFO [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T21:24:07,127 INFO [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T21:24:07,128 INFO [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T21:24:07,132 INFO [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a313eea8709e%2C46445%2C1731878642828.meta, suffix=.meta, logDir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/WALs/a313eea8709e,46445,1731878642828, archiveDir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/oldWALs, maxLogs=32 2024-11-17T21:24:07,147 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/WALs/a313eea8709e,46445,1731878642828/a313eea8709e%2C46445%2C1731878642828.meta.1731878647134.meta, exclude list is [], retry=0 2024-11-17T21:24:07,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35609,DS-df620682-9f57-47c1-b861-3d6acafd468f,DISK] 2024-11-17T21:24:07,153 INFO [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/WALs/a313eea8709e,46445,1731878642828/a313eea8709e%2C46445%2C1731878642828.meta.1731878647134.meta 2024-11-17T21:24:07,154 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43631:43631)] 2024-11-17T21:24:07,154 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T21:24:07,156 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T21:24:07,158 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T21:24:07,162 INFO [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T21:24:07,166 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T21:24:07,166 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:24:07,167 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T21:24:07,167 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T21:24:07,170 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T21:24:07,171 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T21:24:07,171 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:07,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T21:24:07,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T21:24:07,174 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T21:24:07,174 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:07,175 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T21:24:07,175 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T21:24:07,176 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T21:24:07,176 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:07,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T21:24:07,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T21:24:07,179 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T21:24:07,179 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:07,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T21:24:07,180 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T21:24:07,182 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740 2024-11-17T21:24:07,185 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740 2024-11-17T21:24:07,188 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T21:24:07,188 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T21:24:07,189 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T21:24:07,192 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T21:24:07,194 INFO [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61371962, jitterRate=-0.085486501455307}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T21:24:07,194 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T21:24:07,196 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731878647167Writing region info on filesystem at 1731878647167Initializing all the Stores at 1731878647169 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731878647169Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731878647170 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878647170Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731878647170Cleaning up temporary data from old regions at 1731878647188 (+18 ms)Running coprocessor post-open hooks at 1731878647194 (+6 ms)Region opened successfully at 1731878647196 (+2 ms) 2024-11-17T21:24:07,202 INFO [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731878647104 2024-11-17T21:24:07,212 DEBUG [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T21:24:07,212 INFO [RS_OPEN_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T21:24:07,214 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:24:07,216 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a313eea8709e,46445,1731878642828, state=OPEN 2024-11-17T21:24:07,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T21:24:07,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T21:24:07,309 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T21:24:07,309 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T21:24:07,310 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:07,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-17T21:24:07,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a313eea8709e,46445,1731878642828 in 377 msec 2024-11-17T21:24:07,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-17T21:24:07,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 674 msec 2024-11-17T21:24:07,328 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T21:24:07,328 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-17T21:24:07,346 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:07,347 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:07,364 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:07,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52781, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:07,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7300 sec 2024-11-17T21:24:07,387 INFO [master/a313eea8709e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731878647387, completionTime=-1 2024-11-17T21:24:07,390 INFO [master/a313eea8709e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-17T21:24:07,390 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-17T21:24:07,414 INFO [master/a313eea8709e:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-17T21:24:07,414 INFO [master/a313eea8709e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731878707414 2024-11-17T21:24:07,414 INFO [master/a313eea8709e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731878767414 2024-11-17T21:24:07,414 INFO [master/a313eea8709e:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 24 msec 2024-11-17T21:24:07,417 INFO [master/a313eea8709e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a313eea8709e,43623,1731878641945-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:07,417 INFO [master/a313eea8709e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a313eea8709e,43623,1731878641945-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:07,417 INFO [master/a313eea8709e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a313eea8709e,43623,1731878641945-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:07,419 INFO [master/a313eea8709e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a313eea8709e:43623, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:07,419 INFO [master/a313eea8709e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:07,420 INFO [master/a313eea8709e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:07,425 DEBUG [master/a313eea8709e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-17T21:24:07,451 INFO [master/a313eea8709e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.391sec 2024-11-17T21:24:07,452 INFO [master/a313eea8709e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-17T21:24:07,454 INFO [master/a313eea8709e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-17T21:24:07,455 INFO [master/a313eea8709e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-17T21:24:07,456 INFO [master/a313eea8709e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-17T21:24:07,456 INFO [master/a313eea8709e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-17T21:24:07,457 INFO [master/a313eea8709e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a313eea8709e,43623,1731878641945-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T21:24:07,457 INFO [master/a313eea8709e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a313eea8709e,43623,1731878641945-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-17T21:24:07,470 DEBUG [master/a313eea8709e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-17T21:24:07,471 INFO [master/a313eea8709e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-17T21:24:07,472 INFO [master/a313eea8709e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a313eea8709e,43623,1731878641945-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T21:24:07,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519673c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:07,490 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-17T21:24:07,490 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-17T21:24:07,494 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:07,496 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:07,504 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d6faf04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:07,514 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:07,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2d93e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:07,517 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:07,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:07,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a313eea8709e,43623,1731878641945 2024-11-17T21:24:07,567 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=361, ProcessCount=12, AvailableMemoryMB=9490 2024-11-17T21:24:07,576 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T21:24:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a313eea8709e,43623,1731878641945 2024-11-17T21:24:07,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5bad00f6 2024-11-17T21:24:07,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T21:24:07,588 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T21:24:07,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T21:24:07,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T21:24:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T21:24:07,609 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T21:24:07,626 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-17T21:24:07,626 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:07,629 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T21:24:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T21:24:07,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741835_1011 (size=959) 2024-11-17T21:24:07,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T21:24:07,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T21:24:08,050 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249 2024-11-17T21:24:08,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741836_1012 (size=53) 2024-11-17T21:24:08,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T21:24:08,460 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:24:08,460 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8417a211937373f3c01713928c5cb18b, disabling compactions & flushes 2024-11-17T21:24:08,460 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:08,460 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:08,460 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. after waiting 0 ms 2024-11-17T21:24:08,461 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:08,461 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:08,461 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8417a211937373f3c01713928c5cb18b: Waiting for close lock at 1731878648460Disabling compacts and flushes for region at 1731878648460Disabling writes for close at 1731878648461 (+1 ms)Writing region close event to WAL at 1731878648461Closed at 1731878648461 2024-11-17T21:24:08,464 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T21:24:08,470 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731878648464"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731878648464"}]},"ts":"1731878648464"} 2024-11-17T21:24:08,476 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T21:24:08,478 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T21:24:08,481 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878648479"}]},"ts":"1731878648479"} 2024-11-17T21:24:08,487 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T21:24:08,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8417a211937373f3c01713928c5cb18b, ASSIGN}] 2024-11-17T21:24:08,493 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8417a211937373f3c01713928c5cb18b, ASSIGN 2024-11-17T21:24:08,495 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8417a211937373f3c01713928c5cb18b, ASSIGN; state=OFFLINE, location=a313eea8709e,46445,1731878642828; forceNewPlan=false, retain=false 2024-11-17T21:24:08,648 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8417a211937373f3c01713928c5cb18b, regionState=OPENING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:24:08,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8417a211937373f3c01713928c5cb18b, ASSIGN because future has completed 2024-11-17T21:24:08,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:24:08,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T21:24:08,814 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:08,815 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} 2024-11-17T21:24:08,816 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:08,816 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:24:08,817 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:08,817 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:08,819 INFO [StoreOpener-8417a211937373f3c01713928c5cb18b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:08,829 INFO [StoreOpener-8417a211937373f3c01713928c5cb18b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8417a211937373f3c01713928c5cb18b columnFamilyName A 2024-11-17T21:24:08,829 DEBUG [StoreOpener-8417a211937373f3c01713928c5cb18b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:08,831 INFO [StoreOpener-8417a211937373f3c01713928c5cb18b-1 {}] regionserver.HStore(327): Store=8417a211937373f3c01713928c5cb18b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:24:08,831 INFO [StoreOpener-8417a211937373f3c01713928c5cb18b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:08,835 INFO [StoreOpener-8417a211937373f3c01713928c5cb18b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8417a211937373f3c01713928c5cb18b columnFamilyName B 2024-11-17T21:24:08,835 DEBUG [StoreOpener-8417a211937373f3c01713928c5cb18b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:08,836 INFO [StoreOpener-8417a211937373f3c01713928c5cb18b-1 {}] regionserver.HStore(327): Store=8417a211937373f3c01713928c5cb18b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:24:08,836 INFO [StoreOpener-8417a211937373f3c01713928c5cb18b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:08,840 INFO [StoreOpener-8417a211937373f3c01713928c5cb18b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8417a211937373f3c01713928c5cb18b columnFamilyName C 2024-11-17T21:24:08,840 DEBUG [StoreOpener-8417a211937373f3c01713928c5cb18b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:08,841 INFO [StoreOpener-8417a211937373f3c01713928c5cb18b-1 {}] regionserver.HStore(327): Store=8417a211937373f3c01713928c5cb18b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:24:08,841 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:08,843 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:08,844 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:08,846 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:08,847 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:08,847 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T21:24:08,850 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:08,857 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T21:24:08,859 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8417a211937373f3c01713928c5cb18b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68065699, jitterRate=0.014257952570915222}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T21:24:08,859 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:08,860 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8417a211937373f3c01713928c5cb18b: Running coprocessor pre-open hook at 1731878648817Writing region info on filesystem at 1731878648817Initializing all the Stores at 1731878648819 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878648819Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878648819Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878648819Cleaning up temporary data from old regions at 1731878648847 (+28 ms)Running coprocessor post-open hooks at 1731878648859 (+12 ms)Region opened successfully at 1731878648860 (+1 ms) 2024-11-17T21:24:08,862 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., pid=6, masterSystemTime=1731878648807 2024-11-17T21:24:08,867 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:08,867 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:08,868 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8417a211937373f3c01713928c5cb18b, regionState=OPEN, openSeqNum=2, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:24:08,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:24:08,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-17T21:24:08,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 in 226 msec 2024-11-17T21:24:08,893 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-17T21:24:08,893 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8417a211937373f3c01713928c5cb18b, ASSIGN in 397 msec 2024-11-17T21:24:08,895 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T21:24:08,896 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878648895"}]},"ts":"1731878648895"} 2024-11-17T21:24:08,900 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T21:24:08,903 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T21:24:08,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3030 sec 2024-11-17T21:24:09,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T21:24:09,783 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:09,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5ede62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:09,790 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:09,791 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596504f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:09,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:09,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:09,795 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f7e812, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,797 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:09,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:09,802 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:09,805 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T21:24:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is a313eea8709e,43623,1731878641945 2024-11-17T21:24:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2702f94 2024-11-17T21:24:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T21:24:09,812 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T21:24:09,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3812e936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:09,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:09,819 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:09,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7290076, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:09,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:09,824 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b17c100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e500c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:09,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:09,829 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f338c72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:09,833 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:09,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4184659e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c12611, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:09,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:09,839 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:09,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:09,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b383f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:09,843 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f55d533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77155d64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,845 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:09,846 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:09,848 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:09,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ccb4eda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:09,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:09,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:09,852 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:09,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e934139, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bdceae5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,855 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:09,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:09,856 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68fd5b02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:09,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:09,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:09,859 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:09,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9807ff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58df653d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,863 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:09,863 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:09,865 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14983026, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:09,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:09,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:09,868 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:09,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41adc76f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85187e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:09,871 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:09,872 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:09,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:09,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526369d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:09,875 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be2e700, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc6fa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:09,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:09,883 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:09,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9f638da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:09,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:09,886 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:09,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ab586a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a990c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,893 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:09,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:09,895 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:09,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:09,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:09,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30b6e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:09,898 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed66a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:09,905 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:09,905 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:09,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:09,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:09,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:09,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:09,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:09,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:09,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:24:09,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:24:09,926 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:09,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:09,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:09,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:09,931 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:09,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:24:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-17T21:24:09,944 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:09,944 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T21:24:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:09,947 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:09,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:09,947 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:09,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:09,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:09,949 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:09,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:09,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:09,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:24:09,954 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:24:09,955 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:09,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:24:09,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:09,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:09,959 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:09,961 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:09,962 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:24:09,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:09,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:09,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:24:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:24:10,027 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:10,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:24:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T21:24:10,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T21:24:10,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:10,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:10,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:10,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/cf96fe7346bd4dba80dd031dbc7a2a94 is 50, key is test_row_0/A:col10/1731878649943/Put/seqid=0 2024-11-17T21:24:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741837_1013 (size=14341) 2024-11-17T21:24:10,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878710191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/cf96fe7346bd4dba80dd031dbc7a2a94 2024-11-17T21:24:10,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878710206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878710206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878710206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878710210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T21:24:10,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T21:24:10,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:10,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:10,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:10,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878710322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1259e240cff849758d2e66eb1abc5140 is 50, key is test_row_0/B:col10/1731878649943/Put/seqid=0 2024-11-17T21:24:10,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878710336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878710341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878710341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878710343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741838_1014 (size=12001) 2024-11-17T21:24:10,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1259e240cff849758d2e66eb1abc5140 2024-11-17T21:24:10,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/3bcf53c78a214ffb8a903d380027872c is 50, key is test_row_0/C:col10/1731878649943/Put/seqid=0 2024-11-17T21:24:10,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741839_1015 (size=12001) 2024-11-17T21:24:10,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T21:24:10,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:10,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:10,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:10,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878710553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878710554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878710554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878710554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878710566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T21:24:10,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T21:24:10,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:10,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:10,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:10,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T21:24:10,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:10,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:10,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:10,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878710862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878710862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878710872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878710872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/3bcf53c78a214ffb8a903d380027872c 2024-11-17T21:24:10,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878710882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:10,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/cf96fe7346bd4dba80dd031dbc7a2a94 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/cf96fe7346bd4dba80dd031dbc7a2a94 2024-11-17T21:24:10,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/cf96fe7346bd4dba80dd031dbc7a2a94, entries=200, sequenceid=12, filesize=14.0 K 2024-11-17T21:24:10,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1259e240cff849758d2e66eb1abc5140 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1259e240cff849758d2e66eb1abc5140 2024-11-17T21:24:10,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1259e240cff849758d2e66eb1abc5140, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T21:24:10,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/3bcf53c78a214ffb8a903d380027872c as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/3bcf53c78a214ffb8a903d380027872c 2024-11-17T21:24:10,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T21:24:10,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:10,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:10,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:10,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:10,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/3bcf53c78a214ffb8a903d380027872c, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T21:24:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:11,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8417a211937373f3c01713928c5cb18b in 955ms, sequenceid=12, compaction requested=false 2024-11-17T21:24:11,006 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T21:24:11,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T21:24:11,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T21:24:11,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:11,146 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T21:24:11,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/fa8d1d43d29c4eceb16372cb60628e78 is 50, key is test_row_0/A:col10/1731878650189/Put/seqid=0 2024-11-17T21:24:11,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741840_1016 (size=12001) 2024-11-17T21:24:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:11,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:11,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:11,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878711401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878711405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:11,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:11,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878711405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:11,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:11,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878711414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:11,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878711414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:11,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878711522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:11,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878711522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:11,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878711532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:11,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:11,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878711532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:11,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:11,594 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/fa8d1d43d29c4eceb16372cb60628e78 2024-11-17T21:24:11,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/55bdfe53ccf84a269bd32023b1e00ec7 is 50, key is test_row_0/B:col10/1731878650189/Put/seqid=0 2024-11-17T21:24:11,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741841_1017 (size=12001) 2024-11-17T21:24:11,649 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/55bdfe53ccf84a269bd32023b1e00ec7 2024-11-17T21:24:11,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/0f36ea8f282b4994acb696e07799ac83 is 50, key is test_row_0/C:col10/1731878650189/Put/seqid=0 2024-11-17T21:24:11,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741842_1018 (size=12001) 2024-11-17T21:24:11,689 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/0f36ea8f282b4994acb696e07799ac83 2024-11-17T21:24:11,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/fa8d1d43d29c4eceb16372cb60628e78 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/fa8d1d43d29c4eceb16372cb60628e78 2024-11-17T21:24:11,734 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/fa8d1d43d29c4eceb16372cb60628e78, entries=150, sequenceid=37, filesize=11.7 K 2024-11-17T21:24:11,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:11,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878711732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878711732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:11,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:11,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/55bdfe53ccf84a269bd32023b1e00ec7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/55bdfe53ccf84a269bd32023b1e00ec7 2024-11-17T21:24:11,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878711742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:11,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:11,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878711751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:11,757 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/55bdfe53ccf84a269bd32023b1e00ec7, entries=150, sequenceid=37, filesize=11.7 K 2024-11-17T21:24:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/0f36ea8f282b4994acb696e07799ac83 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/0f36ea8f282b4994acb696e07799ac83 2024-11-17T21:24:11,778 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/0f36ea8f282b4994acb696e07799ac83, entries=150, sequenceid=37, filesize=11.7 K 2024-11-17T21:24:11,780 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 8417a211937373f3c01713928c5cb18b in 635ms, sequenceid=37, compaction requested=false 2024-11-17T21:24:11,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:11,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:11,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-17T21:24:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-17T21:24:11,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-17T21:24:11,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8350 sec 2024-11-17T21:24:11,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.8600 sec 2024-11-17T21:24:11,952 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-17T21:24:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:12,046 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:12,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T21:24:12,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/ca2a994bdbbb462a8208b5e4889e9a3a is 50, key is test_row_0/A:col10/1731878652043/Put/seqid=0 2024-11-17T21:24:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T21:24:12,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741843_1019 (size=12001) 2024-11-17T21:24:12,091 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:12,092 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:12,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/ca2a994bdbbb462a8208b5e4889e9a3a 2024-11-17T21:24:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-17T21:24:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T21:24:12,097 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:12,099 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:12,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:12,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878712109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878712116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/de2f4cd2d0c14e71a8d217506c9093dd is 50, key is test_row_0/B:col10/1731878652043/Put/seqid=0 2024-11-17T21:24:12,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878712119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878712121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741844_1020 (size=12001) 2024-11-17T21:24:12,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/de2f4cd2d0c14e71a8d217506c9093dd 2024-11-17T21:24:12,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/1704fd7fde514c14bc58582e482a0f9f is 50, key is test_row_0/C:col10/1731878652043/Put/seqid=0 2024-11-17T21:24:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T21:24:12,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741845_1021 (size=12001) 2024-11-17T21:24:12,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/1704fd7fde514c14bc58582e482a0f9f 2024-11-17T21:24:12,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/ca2a994bdbbb462a8208b5e4889e9a3a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ca2a994bdbbb462a8208b5e4889e9a3a 2024-11-17T21:24:12,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878712232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ca2a994bdbbb462a8208b5e4889e9a3a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T21:24:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878712233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878712233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878712234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/de2f4cd2d0c14e71a8d217506c9093dd as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/de2f4cd2d0c14e71a8d217506c9093dd 2024-11-17T21:24:12,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T21:24:12,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:12,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:12,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:12,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:12,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:12,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/de2f4cd2d0c14e71a8d217506c9093dd, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T21:24:12,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/1704fd7fde514c14bc58582e482a0f9f as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1704fd7fde514c14bc58582e482a0f9f 2024-11-17T21:24:12,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1704fd7fde514c14bc58582e482a0f9f, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T21:24:12,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 8417a211937373f3c01713928c5cb18b in 241ms, sequenceid=51, compaction requested=true 2024-11-17T21:24:12,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:12,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:12,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:12,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:12,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:12,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:12,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:12,304 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:12,304 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:12,312 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:12,312 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:12,314 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/A is initiating minor compaction (all files) 2024-11-17T21:24:12,314 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/A in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:12,315 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/B is initiating minor compaction (all files) 2024-11-17T21:24:12,315 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/B in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:12,315 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1259e240cff849758d2e66eb1abc5140, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/55bdfe53ccf84a269bd32023b1e00ec7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/de2f4cd2d0c14e71a8d217506c9093dd] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=35.2 K 2024-11-17T21:24:12,315 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/cf96fe7346bd4dba80dd031dbc7a2a94, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/fa8d1d43d29c4eceb16372cb60628e78, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ca2a994bdbbb462a8208b5e4889e9a3a] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=37.4 K 2024-11-17T21:24:12,317 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 1259e240cff849758d2e66eb1abc5140, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731878649943 2024-11-17T21:24:12,317 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf96fe7346bd4dba80dd031dbc7a2a94, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731878649943 2024-11-17T21:24:12,319 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 55bdfe53ccf84a269bd32023b1e00ec7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731878650181 2024-11-17T21:24:12,321 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa8d1d43d29c4eceb16372cb60628e78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731878650181 2024-11-17T21:24:12,321 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting de2f4cd2d0c14e71a8d217506c9093dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878651404 2024-11-17T21:24:12,322 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca2a994bdbbb462a8208b5e4889e9a3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878651404 2024-11-17T21:24:12,367 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#A#compaction#9 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:12,369 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/dc56e80ac0ec4924b42ca73d2d8924f6 is 50, key is test_row_0/A:col10/1731878652043/Put/seqid=0 2024-11-17T21:24:12,371 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#B#compaction#10 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:12,372 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/8985412ee2a944d5a41ad36a3dbc5869 is 50, key is test_row_0/B:col10/1731878652043/Put/seqid=0 2024-11-17T21:24:12,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741846_1022 (size=12104) 2024-11-17T21:24:12,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T21:24:12,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T21:24:12,411 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T21:24:12,419 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/dc56e80ac0ec4924b42ca73d2d8924f6 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/dc56e80ac0ec4924b42ca73d2d8924f6 2024-11-17T21:24:12,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741847_1023 (size=12104) 2024-11-17T21:24:12,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/8816da2ca1af411ab4c274e9198ea6d1 is 50, key is test_row_0/A:col10/1731878652112/Put/seqid=0 2024-11-17T21:24:12,455 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/8985412ee2a944d5a41ad36a3dbc5869 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/8985412ee2a944d5a41ad36a3dbc5869 2024-11-17T21:24:12,460 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/A of 8417a211937373f3c01713928c5cb18b into dc56e80ac0ec4924b42ca73d2d8924f6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:12,460 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:12,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:12,463 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/A, priority=13, startTime=1731878652289; duration=0sec 2024-11-17T21:24:12,464 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:12,464 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:A 2024-11-17T21:24:12,464 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:12,470 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/B of 8417a211937373f3c01713928c5cb18b into 8985412ee2a944d5a41ad36a3dbc5869(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:12,470 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:12,470 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/B, priority=13, startTime=1731878652303; duration=0sec 2024-11-17T21:24:12,470 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:12,471 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:B 2024-11-17T21:24:12,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741848_1024 (size=12001) 2024-11-17T21:24:12,473 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:12,473 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/8816da2ca1af411ab4c274e9198ea6d1 2024-11-17T21:24:12,473 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/C is initiating minor compaction (all files) 2024-11-17T21:24:12,473 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/C in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:12,474 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/3bcf53c78a214ffb8a903d380027872c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/0f36ea8f282b4994acb696e07799ac83, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1704fd7fde514c14bc58582e482a0f9f] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=35.2 K 2024-11-17T21:24:12,475 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bcf53c78a214ffb8a903d380027872c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731878649943 2024-11-17T21:24:12,476 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f36ea8f282b4994acb696e07799ac83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731878650181 2024-11-17T21:24:12,476 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1704fd7fde514c14bc58582e482a0f9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878651404 2024-11-17T21:24:12,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/be5d03076c324d3a8499fb9fad634c32 is 50, key is test_row_0/B:col10/1731878652112/Put/seqid=0 2024-11-17T21:24:12,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878712501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,527 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#C#compaction#13 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:12,528 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/3d63f573c2874a538d1b123b456d5406 is 50, key is test_row_0/C:col10/1731878652043/Put/seqid=0 2024-11-17T21:24:12,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878712506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878712521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878712522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878712532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741849_1025 (size=12001) 2024-11-17T21:24:12,553 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/be5d03076c324d3a8499fb9fad634c32 2024-11-17T21:24:12,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741850_1026 (size=12104) 2024-11-17T21:24:12,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/4a4161a33d024c568a5735a1a4f0b934 is 50, key is test_row_0/C:col10/1731878652112/Put/seqid=0 2024-11-17T21:24:12,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741851_1027 (size=12001) 2024-11-17T21:24:12,593 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/4a4161a33d024c568a5735a1a4f0b934 2024-11-17T21:24:12,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/8816da2ca1af411ab4c274e9198ea6d1 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8816da2ca1af411ab4c274e9198ea6d1 2024-11-17T21:24:12,622 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8816da2ca1af411ab4c274e9198ea6d1, entries=150, sequenceid=73, filesize=11.7 K 2024-11-17T21:24:12,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/be5d03076c324d3a8499fb9fad634c32 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/be5d03076c324d3a8499fb9fad634c32 2024-11-17T21:24:12,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878712632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,641 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/be5d03076c324d3a8499fb9fad634c32, entries=150, sequenceid=73, filesize=11.7 K 2024-11-17T21:24:12,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/4a4161a33d024c568a5735a1a4f0b934 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/4a4161a33d024c568a5735a1a4f0b934 2024-11-17T21:24:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878712642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878712652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878712651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878712661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,667 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/4a4161a33d024c568a5735a1a4f0b934, entries=150, sequenceid=73, filesize=11.7 K 2024-11-17T21:24:12,669 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=87.22 KB/89310 for 8417a211937373f3c01713928c5cb18b in 258ms, sequenceid=73, compaction requested=false 2024-11-17T21:24:12,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:12,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:12,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-17T21:24:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-17T21:24:12,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-17T21:24:12,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 573 msec 2024-11-17T21:24:12,684 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T21:24:12,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 593 msec 2024-11-17T21:24:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T21:24:12,721 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:12,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:12,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-17T21:24:12,726 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T21:24:12,728 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:12,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:12,788 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-17T21:24:12,789 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-17T21:24:12,791 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T21:24:12,791 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-17T21:24:12,791 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-17T21:24:12,791 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-17T21:24:12,793 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T21:24:12,793 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T21:24:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T21:24:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:12,845 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:12,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-17T21:24:12,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/ed69eecc2d14452b85c4e9ae96d8a939 is 50, key is test_row_0/A:col10/1731878652842/Put/seqid=0 2024-11-17T21:24:12,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741852_1028 (size=14341) 2024-11-17T21:24:12,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T21:24:12,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:12,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:12,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:12,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:12,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/ed69eecc2d14452b85c4e9ae96d8a939 2024-11-17T21:24:12,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:12,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:12,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/23b39a5291f34f80b7dd560ba0357626 is 50, key is test_row_0/B:col10/1731878652842/Put/seqid=0 2024-11-17T21:24:12,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878712896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878712901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878712896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878712906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878712907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:12,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:12,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:12,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741853_1029 (size=12001) 2024-11-17T21:24:12,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/23b39a5291f34f80b7dd560ba0357626 2024-11-17T21:24:12,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/a037c0be97274a86bd9478e1c2325d06 is 50, key is test_row_0/C:col10/1731878652842/Put/seqid=0 2024-11-17T21:24:12,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741854_1030 (size=12001) 2024-11-17T21:24:12,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/a037c0be97274a86bd9478e1c2325d06 2024-11-17T21:24:12,992 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/3d63f573c2874a538d1b123b456d5406 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/3d63f573c2874a538d1b123b456d5406 2024-11-17T21:24:13,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/ed69eecc2d14452b85c4e9ae96d8a939 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ed69eecc2d14452b85c4e9ae96d8a939 2024-11-17T21:24:13,009 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/C of 8417a211937373f3c01713928c5cb18b into 3d63f573c2874a538d1b123b456d5406(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:13,009 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:13,009 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/C, priority=13, startTime=1731878652306; duration=0sec 2024-11-17T21:24:13,010 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:13,010 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:C 2024-11-17T21:24:13,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ed69eecc2d14452b85c4e9ae96d8a939, entries=200, sequenceid=92, filesize=14.0 K 2024-11-17T21:24:13,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/23b39a5291f34f80b7dd560ba0357626 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/23b39a5291f34f80b7dd560ba0357626 2024-11-17T21:24:13,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878713022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878713022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878713022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878713022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878713022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T21:24:13,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:13,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:13,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:13,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:13,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:13,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:13,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/23b39a5291f34f80b7dd560ba0357626, entries=150, sequenceid=92, filesize=11.7 K 2024-11-17T21:24:13,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T21:24:13,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/a037c0be97274a86bd9478e1c2325d06 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a037c0be97274a86bd9478e1c2325d06 2024-11-17T21:24:13,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a037c0be97274a86bd9478e1c2325d06, entries=150, sequenceid=92, filesize=11.7 K 2024-11-17T21:24:13,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 8417a211937373f3c01713928c5cb18b in 220ms, sequenceid=92, compaction requested=true 2024-11-17T21:24:13,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:13,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:13,066 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:13,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:13,067 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:13,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:13,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:13,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:13,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:13,068 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:13,068 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/A is initiating minor compaction (all files) 2024-11-17T21:24:13,068 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/A in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:13,068 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/dc56e80ac0ec4924b42ca73d2d8924f6, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8816da2ca1af411ab4c274e9198ea6d1, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ed69eecc2d14452b85c4e9ae96d8a939] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=37.5 K 2024-11-17T21:24:13,069 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:13,069 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/B is initiating minor compaction (all files) 2024-11-17T21:24:13,069 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/B in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:13,069 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/8985412ee2a944d5a41ad36a3dbc5869, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/be5d03076c324d3a8499fb9fad634c32, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/23b39a5291f34f80b7dd560ba0357626] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=35.3 K 2024-11-17T21:24:13,069 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc56e80ac0ec4924b42ca73d2d8924f6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878651404 2024-11-17T21:24:13,070 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 8985412ee2a944d5a41ad36a3dbc5869, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878651404 2024-11-17T21:24:13,070 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8816da2ca1af411ab4c274e9198ea6d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731878652100 2024-11-17T21:24:13,070 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting be5d03076c324d3a8499fb9fad634c32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731878652100 2024-11-17T21:24:13,072 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed69eecc2d14452b85c4e9ae96d8a939, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731878652512 2024-11-17T21:24:13,072 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 23b39a5291f34f80b7dd560ba0357626, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731878652513 2024-11-17T21:24:13,105 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#A#compaction#18 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:13,106 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/3a51f5b5adc941529b62d9fc9278605b is 50, key is test_row_0/A:col10/1731878652842/Put/seqid=0 2024-11-17T21:24:13,119 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#B#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:13,121 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/b6f7f6a22c254a03b8fab0392cd1e8fa is 50, key is test_row_0/B:col10/1731878652842/Put/seqid=0 2024-11-17T21:24:13,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741855_1031 (size=12207) 2024-11-17T21:24:13,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741856_1032 (size=12207) 2024-11-17T21:24:13,178 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/3a51f5b5adc941529b62d9fc9278605b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/3a51f5b5adc941529b62d9fc9278605b 2024-11-17T21:24:13,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T21:24:13,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:13,194 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-17T21:24:13,196 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/b6f7f6a22c254a03b8fab0392cd1e8fa as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/b6f7f6a22c254a03b8fab0392cd1e8fa 2024-11-17T21:24:13,202 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/A of 8417a211937373f3c01713928c5cb18b into 3a51f5b5adc941529b62d9fc9278605b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:13,203 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:13,203 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/A, priority=13, startTime=1731878653065; duration=0sec 2024-11-17T21:24:13,203 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:13,203 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:A 2024-11-17T21:24:13,203 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:13,207 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:13,207 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/C is initiating minor compaction (all files) 2024-11-17T21:24:13,207 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/C in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:13,208 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/3d63f573c2874a538d1b123b456d5406, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/4a4161a33d024c568a5735a1a4f0b934, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a037c0be97274a86bd9478e1c2325d06] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=35.3 K 2024-11-17T21:24:13,213 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d63f573c2874a538d1b123b456d5406, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878651404 2024-11-17T21:24:13,213 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/B of 8417a211937373f3c01713928c5cb18b into b6f7f6a22c254a03b8fab0392cd1e8fa(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:13,213 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:13,213 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/B, priority=13, startTime=1731878653067; duration=0sec 2024-11-17T21:24:13,214 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:13,214 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:B 2024-11-17T21:24:13,214 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a4161a33d024c568a5735a1a4f0b934, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731878652100 2024-11-17T21:24:13,215 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting a037c0be97274a86bd9478e1c2325d06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731878652513 2024-11-17T21:24:13,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/41e207a747de49f1b724d8de9ad675f2 is 50, key is test_row_0/A:col10/1731878652895/Put/seqid=0 2024-11-17T21:24:13,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:13,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:13,251 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#C#compaction#21 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:13,252 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/83b89d7ce86b4141a766b9eda57d5535 is 50, key is test_row_0/C:col10/1731878652842/Put/seqid=0 2024-11-17T21:24:13,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741857_1033 (size=12001) 2024-11-17T21:24:13,268 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/41e207a747de49f1b724d8de9ad675f2 2024-11-17T21:24:13,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878713268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878713269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878713271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878713274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878713274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1f4f2a40aa4143378218f9e2015b2f1b is 50, key is test_row_0/B:col10/1731878652895/Put/seqid=0 2024-11-17T21:24:13,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741858_1034 (size=12207) 2024-11-17T21:24:13,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741859_1035 (size=12001) 2024-11-17T21:24:13,322 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1f4f2a40aa4143378218f9e2015b2f1b 2024-11-17T21:24:13,323 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/83b89d7ce86b4141a766b9eda57d5535 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/83b89d7ce86b4141a766b9eda57d5535 2024-11-17T21:24:13,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/cd00ff8d07d840aaaff71129ba978baf is 50, key is test_row_0/C:col10/1731878652895/Put/seqid=0 2024-11-17T21:24:13,341 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/C of 8417a211937373f3c01713928c5cb18b into 83b89d7ce86b4141a766b9eda57d5535(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:13,341 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:13,341 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/C, priority=13, startTime=1731878653067; duration=0sec 2024-11-17T21:24:13,341 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:13,342 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:C 2024-11-17T21:24:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T21:24:13,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741860_1036 (size=12001) 2024-11-17T21:24:13,355 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/cd00ff8d07d840aaaff71129ba978baf 2024-11-17T21:24:13,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/41e207a747de49f1b724d8de9ad675f2 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/41e207a747de49f1b724d8de9ad675f2 2024-11-17T21:24:13,381 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/41e207a747de49f1b724d8de9ad675f2, entries=150, sequenceid=112, filesize=11.7 K 2024-11-17T21:24:13,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1f4f2a40aa4143378218f9e2015b2f1b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1f4f2a40aa4143378218f9e2015b2f1b 2024-11-17T21:24:13,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878713381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878713392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878713392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878713393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878713393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,398 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1f4f2a40aa4143378218f9e2015b2f1b, entries=150, sequenceid=112, filesize=11.7 K 2024-11-17T21:24:13,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/cd00ff8d07d840aaaff71129ba978baf as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/cd00ff8d07d840aaaff71129ba978baf 2024-11-17T21:24:13,433 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/cd00ff8d07d840aaaff71129ba978baf, entries=150, sequenceid=112, filesize=11.7 K 2024-11-17T21:24:13,435 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 8417a211937373f3c01713928c5cb18b in 240ms, sequenceid=112, compaction requested=false 2024-11-17T21:24:13,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:13,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:13,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-17T21:24:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-17T21:24:13,442 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-17T21:24:13,442 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 709 msec 2024-11-17T21:24:13,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 723 msec 2024-11-17T21:24:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:13,595 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:13,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-17T21:24:13,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/a7bc0b566bb54ae3bb73199fc7dd4732 is 50, key is test_row_0/A:col10/1731878653592/Put/seqid=0 2024-11-17T21:24:13,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741861_1037 (size=14441) 2024-11-17T21:24:13,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878713629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878713629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878713664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878713664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878713665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878713771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878713771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878713780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878713781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878713781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T21:24:13,861 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:13,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-17T21:24:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T21:24:13,867 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:13,873 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:13,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T21:24:13,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878713981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878713981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878713991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878713992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878713992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:13,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:13,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:13,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T21:24:14,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:14,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:14,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:14,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:14,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:14,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/a7bc0b566bb54ae3bb73199fc7dd4732 2024-11-17T21:24:14,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/d65e680d6e7843c9a851e9748ce670f1 is 50, key is test_row_0/B:col10/1731878653592/Put/seqid=0 2024-11-17T21:24:14,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741862_1038 (size=12051) 2024-11-17T21:24:14,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/d65e680d6e7843c9a851e9748ce670f1 2024-11-17T21:24:14,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/1f7a86417d2c4a199b1e1fbf702de61a is 50, key is test_row_0/C:col10/1731878653592/Put/seqid=0 2024-11-17T21:24:14,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741863_1039 (size=12051) 2024-11-17T21:24:14,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/1f7a86417d2c4a199b1e1fbf702de61a 2024-11-17T21:24:14,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/a7bc0b566bb54ae3bb73199fc7dd4732 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/a7bc0b566bb54ae3bb73199fc7dd4732 2024-11-17T21:24:14,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/a7bc0b566bb54ae3bb73199fc7dd4732, entries=200, sequenceid=133, filesize=14.1 K 2024-11-17T21:24:14,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/d65e680d6e7843c9a851e9748ce670f1 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/d65e680d6e7843c9a851e9748ce670f1 2024-11-17T21:24:14,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/d65e680d6e7843c9a851e9748ce670f1, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T21:24:14,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/1f7a86417d2c4a199b1e1fbf702de61a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1f7a86417d2c4a199b1e1fbf702de61a 2024-11-17T21:24:14,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1f7a86417d2c4a199b1e1fbf702de61a, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T21:24:14,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 8417a211937373f3c01713928c5cb18b in 566ms, sequenceid=133, compaction requested=true 2024-11-17T21:24:14,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:14,162 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:14,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:14,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:14,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:14,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:14,162 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:14,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:14,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:14,164 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:14,164 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/A is initiating minor compaction (all files) 2024-11-17T21:24:14,164 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/A in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:14,164 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/3a51f5b5adc941529b62d9fc9278605b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/41e207a747de49f1b724d8de9ad675f2, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/a7bc0b566bb54ae3bb73199fc7dd4732] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=37.7 K 2024-11-17T21:24:14,165 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:14,166 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/B is initiating minor compaction (all files) 2024-11-17T21:24:14,166 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/B in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:14,166 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/b6f7f6a22c254a03b8fab0392cd1e8fa, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1f4f2a40aa4143378218f9e2015b2f1b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/d65e680d6e7843c9a851e9748ce670f1] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=35.4 K 2024-11-17T21:24:14,166 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a51f5b5adc941529b62d9fc9278605b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731878652513 2024-11-17T21:24:14,167 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting b6f7f6a22c254a03b8fab0392cd1e8fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731878652513 2024-11-17T21:24:14,168 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41e207a747de49f1b724d8de9ad675f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731878652895 2024-11-17T21:24:14,169 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7bc0b566bb54ae3bb73199fc7dd4732, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731878653253 2024-11-17T21:24:14,169 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f4f2a40aa4143378218f9e2015b2f1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731878652895 2024-11-17T21:24:14,170 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d65e680d6e7843c9a851e9748ce670f1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731878653253 2024-11-17T21:24:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T21:24:14,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T21:24:14,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:14,184 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-17T21:24:14,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/f79d1b75035c456ca42fd0cf97644894 is 50, key is test_row_0/A:col10/1731878653639/Put/seqid=0 2024-11-17T21:24:14,211 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:14,212 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/6c0493f65e074be59cca38be28d7ff62 is 50, key is test_row_0/A:col10/1731878653592/Put/seqid=0 2024-11-17T21:24:14,214 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#B#compaction#29 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:14,215 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/5b8b43ab925c4279b6309cc6f878955a is 50, key is test_row_0/B:col10/1731878653592/Put/seqid=0 2024-11-17T21:24:14,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741864_1040 (size=12151) 2024-11-17T21:24:14,233 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/f79d1b75035c456ca42fd0cf97644894 2024-11-17T21:24:14,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741865_1041 (size=12359) 2024-11-17T21:24:14,264 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/6c0493f65e074be59cca38be28d7ff62 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6c0493f65e074be59cca38be28d7ff62 2024-11-17T21:24:14,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/7e4ab2b2b39243a88e5c2982ebf80ea5 is 50, key is test_row_0/B:col10/1731878653639/Put/seqid=0 2024-11-17T21:24:14,277 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/A of 8417a211937373f3c01713928c5cb18b into 6c0493f65e074be59cca38be28d7ff62(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:14,278 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:14,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741866_1042 (size=12359) 2024-11-17T21:24:14,278 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/A, priority=13, startTime=1731878654161; duration=0sec 2024-11-17T21:24:14,278 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:14,279 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:A 2024-11-17T21:24:14,279 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:14,284 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:14,284 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/C is initiating minor compaction (all files) 2024-11-17T21:24:14,284 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/C in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:14,285 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/83b89d7ce86b4141a766b9eda57d5535, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/cd00ff8d07d840aaaff71129ba978baf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1f7a86417d2c4a199b1e1fbf702de61a] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=35.4 K 2024-11-17T21:24:14,286 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83b89d7ce86b4141a766b9eda57d5535, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731878652513 2024-11-17T21:24:14,288 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd00ff8d07d840aaaff71129ba978baf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731878652895 2024-11-17T21:24:14,288 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f7a86417d2c4a199b1e1fbf702de61a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731878653253 2024-11-17T21:24:14,293 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/5b8b43ab925c4279b6309cc6f878955a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/5b8b43ab925c4279b6309cc6f878955a 2024-11-17T21:24:14,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:14,305 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/B of 8417a211937373f3c01713928c5cb18b into 5b8b43ab925c4279b6309cc6f878955a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:14,305 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:14,305 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/B, priority=13, startTime=1731878654162; duration=0sec 2024-11-17T21:24:14,305 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:14,305 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:B 2024-11-17T21:24:14,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741867_1043 (size=12151) 2024-11-17T21:24:14,318 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#C#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:14,319 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/ce2890b8a7be425fb2ec3dcabd0a0f15 is 50, key is test_row_0/C:col10/1731878653592/Put/seqid=0 2024-11-17T21:24:14,322 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/7e4ab2b2b39243a88e5c2982ebf80ea5 2024-11-17T21:24:14,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/bb0ae0dadbc6471f9020df12abad8064 is 50, key is test_row_0/C:col10/1731878653639/Put/seqid=0 2024-11-17T21:24:14,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741868_1044 (size=12359) 2024-11-17T21:24:14,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741869_1045 (size=12151) 2024-11-17T21:24:14,351 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/bb0ae0dadbc6471f9020df12abad8064 2024-11-17T21:24:14,357 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/ce2890b8a7be425fb2ec3dcabd0a0f15 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/ce2890b8a7be425fb2ec3dcabd0a0f15 2024-11-17T21:24:14,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/f79d1b75035c456ca42fd0cf97644894 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/f79d1b75035c456ca42fd0cf97644894 2024-11-17T21:24:14,371 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/C of 8417a211937373f3c01713928c5cb18b into ce2890b8a7be425fb2ec3dcabd0a0f15(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:14,371 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:14,371 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/C, priority=13, startTime=1731878654162; duration=0sec 2024-11-17T21:24:14,371 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:14,371 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:C 2024-11-17T21:24:14,374 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/f79d1b75035c456ca42fd0cf97644894, entries=150, sequenceid=151, filesize=11.9 K 2024-11-17T21:24:14,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/7e4ab2b2b39243a88e5c2982ebf80ea5 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/7e4ab2b2b39243a88e5c2982ebf80ea5 2024-11-17T21:24:14,389 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/7e4ab2b2b39243a88e5c2982ebf80ea5, entries=150, sequenceid=151, filesize=11.9 K 2024-11-17T21:24:14,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/bb0ae0dadbc6471f9020df12abad8064 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/bb0ae0dadbc6471f9020df12abad8064 2024-11-17T21:24:14,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878714379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878714381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878714385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878714388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878714395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,409 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/bb0ae0dadbc6471f9020df12abad8064, entries=150, sequenceid=151, filesize=11.9 K 2024-11-17T21:24:14,412 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 8417a211937373f3c01713928c5cb18b in 228ms, sequenceid=151, compaction requested=false 2024-11-17T21:24:14,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:14,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:14,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-17T21:24:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-17T21:24:14,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-17T21:24:14,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 548 msec 2024-11-17T21:24:14,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 565 msec 2024-11-17T21:24:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T21:24:14,491 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:14,492 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-17T21:24:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T21:24:14,495 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:14,496 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:14,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:14,504 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:14,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T21:24:14,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/a643a105c2d84435b93d6eba61bced63 is 50, key is test_row_0/A:col10/1731878654380/Put/seqid=0 2024-11-17T21:24:14,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878714523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878714526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878714530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878714533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878714537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741870_1046 (size=12151) 2024-11-17T21:24:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T21:24:14,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878714641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878714641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878714641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878714642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878714642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T21:24:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:14,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:14,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:14,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T21:24:14,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:14,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:14,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:14,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:14,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T21:24:14,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878714851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878714851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878714851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878714851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878714851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:14,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/a643a105c2d84435b93d6eba61bced63 2024-11-17T21:24:14,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T21:24:14,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:14,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:14,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:14,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:14,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:14,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1236fa9e6b384b21906fb665f4946185 is 50, key is test_row_0/B:col10/1731878654380/Put/seqid=0 2024-11-17T21:24:15,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741871_1047 (size=12151) 2024-11-17T21:24:15,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T21:24:15,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:15,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:15,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:15,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:15,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T21:24:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:15,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878715161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878715161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:15,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:15,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878715161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:15,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:15,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878715162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878715163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:15,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T21:24:15,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:15,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:15,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:15,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:15,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:15,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1236fa9e6b384b21906fb665f4946185 2024-11-17T21:24:15,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/c84a005c8e424d6b9d0d83c95063e4a5 is 50, key is test_row_0/C:col10/1731878654380/Put/seqid=0 2024-11-17T21:24:15,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T21:24:15,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:15,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:15,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:15,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:15,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:15,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741872_1048 (size=12151) 2024-11-17T21:24:15,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/c84a005c8e424d6b9d0d83c95063e4a5 2024-11-17T21:24:15,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/a643a105c2d84435b93d6eba61bced63 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/a643a105c2d84435b93d6eba61bced63 2024-11-17T21:24:15,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/a643a105c2d84435b93d6eba61bced63, entries=150, sequenceid=175, filesize=11.9 K 2024-11-17T21:24:15,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1236fa9e6b384b21906fb665f4946185 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1236fa9e6b384b21906fb665f4946185 2024-11-17T21:24:15,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1236fa9e6b384b21906fb665f4946185, entries=150, sequenceid=175, filesize=11.9 K 2024-11-17T21:24:15,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/c84a005c8e424d6b9d0d83c95063e4a5 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c84a005c8e424d6b9d0d83c95063e4a5 2024-11-17T21:24:15,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c84a005c8e424d6b9d0d83c95063e4a5, entries=150, sequenceid=175, filesize=11.9 K 2024-11-17T21:24:15,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 8417a211937373f3c01713928c5cb18b in 1001ms, sequenceid=175, compaction requested=true 2024-11-17T21:24:15,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:15,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:15,505 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:15,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:15,505 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:15,508 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:15,508 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:15,509 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/B is initiating minor compaction (all files) 2024-11-17T21:24:15,509 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/A is initiating minor compaction (all files) 2024-11-17T21:24:15,509 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/A in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:15,509 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6c0493f65e074be59cca38be28d7ff62, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/f79d1b75035c456ca42fd0cf97644894, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/a643a105c2d84435b93d6eba61bced63] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=35.8 K 2024-11-17T21:24:15,509 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/B in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:15,510 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/5b8b43ab925c4279b6309cc6f878955a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/7e4ab2b2b39243a88e5c2982ebf80ea5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1236fa9e6b384b21906fb665f4946185] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=35.8 K 2024-11-17T21:24:15,510 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c0493f65e074be59cca38be28d7ff62, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731878653253 2024-11-17T21:24:15,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:15,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:15,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:15,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:15,511 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b8b43ab925c4279b6309cc6f878955a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731878653253 2024-11-17T21:24:15,511 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting f79d1b75035c456ca42fd0cf97644894, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731878653623 2024-11-17T21:24:15,511 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e4ab2b2b39243a88e5c2982ebf80ea5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731878653623 2024-11-17T21:24:15,512 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting a643a105c2d84435b93d6eba61bced63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731878654380 2024-11-17T21:24:15,512 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 1236fa9e6b384b21906fb665f4946185, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731878654380 2024-11-17T21:24:15,531 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:15,532 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/598e6d2248dd4517bfd338910d27d658 is 50, key is test_row_0/B:col10/1731878654380/Put/seqid=0 2024-11-17T21:24:15,537 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:15,538 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/1f0358ce43394ccc95f89d9e10aeda1e is 50, key is test_row_0/A:col10/1731878654380/Put/seqid=0 2024-11-17T21:24:15,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T21:24:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:15,586 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T21:24:15,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741874_1050 (size=12561) 2024-11-17T21:24:15,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741873_1049 (size=12561) 2024-11-17T21:24:15,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/f147b9a21fd34b7099bff28f49f08669 is 50, key is test_row_0/A:col10/1731878654527/Put/seqid=0 2024-11-17T21:24:15,604 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/598e6d2248dd4517bfd338910d27d658 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/598e6d2248dd4517bfd338910d27d658 2024-11-17T21:24:15,618 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/B of 8417a211937373f3c01713928c5cb18b into 598e6d2248dd4517bfd338910d27d658(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:15,619 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:15,619 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/B, priority=13, startTime=1731878655505; duration=0sec 2024-11-17T21:24:15,619 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:15,619 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:B 2024-11-17T21:24:15,619 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:15,621 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:15,621 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/C is initiating minor compaction (all files) 2024-11-17T21:24:15,621 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/C in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:15,622 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/ce2890b8a7be425fb2ec3dcabd0a0f15, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/bb0ae0dadbc6471f9020df12abad8064, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c84a005c8e424d6b9d0d83c95063e4a5] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=35.8 K 2024-11-17T21:24:15,622 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting ce2890b8a7be425fb2ec3dcabd0a0f15, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731878653253 2024-11-17T21:24:15,624 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting bb0ae0dadbc6471f9020df12abad8064, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731878653623 2024-11-17T21:24:15,625 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting c84a005c8e424d6b9d0d83c95063e4a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731878654380 2024-11-17T21:24:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T21:24:15,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741875_1051 (size=12151) 2024-11-17T21:24:15,641 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/f147b9a21fd34b7099bff28f49f08669 2024-11-17T21:24:15,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/0aa8643aea4b46a19c60b4c245fc2cf3 is 50, key is test_row_0/B:col10/1731878654527/Put/seqid=0 2024-11-17T21:24:15,667 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#C#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:15,668 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/d65710282e8b41b98c87f5f6c7006693 is 50, key is test_row_0/C:col10/1731878654380/Put/seqid=0 2024-11-17T21:24:15,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741876_1052 (size=12151) 2024-11-17T21:24:15,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:15,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741877_1053 (size=12561) 2024-11-17T21:24:15,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:15,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878715699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878715701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:15,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:15,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878715706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:15,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:15,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878715708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:15,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878715708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:15,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:15,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878715812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878715812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:15,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:15,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:15,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878715821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:15,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:15,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878715821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878715822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,006 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/1f0358ce43394ccc95f89d9e10aeda1e as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/1f0358ce43394ccc95f89d9e10aeda1e 2024-11-17T21:24:16,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878716022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,024 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/A of 8417a211937373f3c01713928c5cb18b into 1f0358ce43394ccc95f89d9e10aeda1e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:16,024 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:16,024 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/A, priority=13, startTime=1731878655505; duration=0sec 2024-11-17T21:24:16,024 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:16,024 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:A 2024-11-17T21:24:16,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878716022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878716031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878716031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878716032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,081 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/0aa8643aea4b46a19c60b4c245fc2cf3 2024-11-17T21:24:16,113 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/d65710282e8b41b98c87f5f6c7006693 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d65710282e8b41b98c87f5f6c7006693 2024-11-17T21:24:16,126 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/C of 8417a211937373f3c01713928c5cb18b into d65710282e8b41b98c87f5f6c7006693(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:16,126 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:16,126 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/C, priority=13, startTime=1731878655510; duration=0sec 2024-11-17T21:24:16,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/1f08018227ce48ffbc6f0f191b29f9ad is 50, key is test_row_0/C:col10/1731878654527/Put/seqid=0 2024-11-17T21:24:16,126 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:16,126 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:C 2024-11-17T21:24:16,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741878_1054 (size=12151) 2024-11-17T21:24:16,165 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/1f08018227ce48ffbc6f0f191b29f9ad 2024-11-17T21:24:16,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/f147b9a21fd34b7099bff28f49f08669 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/f147b9a21fd34b7099bff28f49f08669 2024-11-17T21:24:16,184 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/f147b9a21fd34b7099bff28f49f08669, entries=150, sequenceid=191, filesize=11.9 K 2024-11-17T21:24:16,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/0aa8643aea4b46a19c60b4c245fc2cf3 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0aa8643aea4b46a19c60b4c245fc2cf3 2024-11-17T21:24:16,199 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0aa8643aea4b46a19c60b4c245fc2cf3, entries=150, sequenceid=191, filesize=11.9 K 2024-11-17T21:24:16,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/1f08018227ce48ffbc6f0f191b29f9ad as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1f08018227ce48ffbc6f0f191b29f9ad 2024-11-17T21:24:16,211 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1f08018227ce48ffbc6f0f191b29f9ad, entries=150, sequenceid=191, filesize=11.9 K 2024-11-17T21:24:16,215 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 8417a211937373f3c01713928c5cb18b in 628ms, sequenceid=191, compaction requested=false 2024-11-17T21:24:16,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:16,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:16,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-17T21:24:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-17T21:24:16,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-17T21:24:16,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7210 sec 2024-11-17T21:24:16,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.7370 sec 2024-11-17T21:24:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:16,332 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:16,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T21:24:16,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/b432554b3c8a4cb1a39b0599b2030812 is 50, key is test_row_0/A:col10/1731878655690/Put/seqid=0 2024-11-17T21:24:16,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741879_1055 (size=12151) 2024-11-17T21:24:16,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878716382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878716385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878716385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878716387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878716397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/b432554b3c8a4cb1a39b0599b2030812 2024-11-17T21:24:16,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1a7b509ba72b4be8944da82de9b6348d is 50, key is test_row_0/B:col10/1731878655690/Put/seqid=0 2024-11-17T21:24:16,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741880_1056 (size=12151) 2024-11-17T21:24:16,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1a7b509ba72b4be8944da82de9b6348d 2024-11-17T21:24:16,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/08ae7c27a9db45a5a50562dc8bcfa83d is 50, key is test_row_0/C:col10/1731878655690/Put/seqid=0 2024-11-17T21:24:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878716501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878716501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878716501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878716501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878716511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741881_1057 (size=12151) 2024-11-17T21:24:16,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/08ae7c27a9db45a5a50562dc8bcfa83d 2024-11-17T21:24:16,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/b432554b3c8a4cb1a39b0599b2030812 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b432554b3c8a4cb1a39b0599b2030812 2024-11-17T21:24:16,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b432554b3c8a4cb1a39b0599b2030812, entries=150, sequenceid=217, filesize=11.9 K 2024-11-17T21:24:16,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1a7b509ba72b4be8944da82de9b6348d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1a7b509ba72b4be8944da82de9b6348d 2024-11-17T21:24:16,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1a7b509ba72b4be8944da82de9b6348d, entries=150, sequenceid=217, filesize=11.9 K 2024-11-17T21:24:16,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/08ae7c27a9db45a5a50562dc8bcfa83d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/08ae7c27a9db45a5a50562dc8bcfa83d 2024-11-17T21:24:16,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/08ae7c27a9db45a5a50562dc8bcfa83d, entries=150, sequenceid=217, filesize=11.9 K 2024-11-17T21:24:16,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 8417a211937373f3c01713928c5cb18b in 244ms, sequenceid=217, compaction requested=true 2024-11-17T21:24:16,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:16,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:16,577 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:16,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:16,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:16,577 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:16,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:16,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:16,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:16,578 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:16,578 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/A is initiating minor compaction (all files) 2024-11-17T21:24:16,579 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/A in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:16,579 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/1f0358ce43394ccc95f89d9e10aeda1e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/f147b9a21fd34b7099bff28f49f08669, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b432554b3c8a4cb1a39b0599b2030812] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=36.0 K 2024-11-17T21:24:16,580 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:16,580 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/B is initiating minor compaction (all files) 2024-11-17T21:24:16,580 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/B in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:16,580 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/598e6d2248dd4517bfd338910d27d658, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0aa8643aea4b46a19c60b4c245fc2cf3, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1a7b509ba72b4be8944da82de9b6348d] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=36.0 K 2024-11-17T21:24:16,581 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f0358ce43394ccc95f89d9e10aeda1e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731878654380 2024-11-17T21:24:16,582 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 598e6d2248dd4517bfd338910d27d658, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731878654380 2024-11-17T21:24:16,582 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting f147b9a21fd34b7099bff28f49f08669, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731878654527 2024-11-17T21:24:16,584 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aa8643aea4b46a19c60b4c245fc2cf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731878654527 2024-11-17T21:24:16,585 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting b432554b3c8a4cb1a39b0599b2030812, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731878655690 2024-11-17T21:24:16,586 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a7b509ba72b4be8944da82de9b6348d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731878655690 2024-11-17T21:24:16,626 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#A#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:16,630 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/b2ae0225c58b4f84bcf5fe6b6ac95595 is 50, key is test_row_0/A:col10/1731878655690/Put/seqid=0 2024-11-17T21:24:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T21:24:16,642 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:16,643 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-17T21:24:16,645 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#B#compaction#46 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:16,646 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/60a59c50175e4c44a7db94bc527aadd5 is 50, key is test_row_0/B:col10/1731878655690/Put/seqid=0 2024-11-17T21:24:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T21:24:16,647 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:16,649 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:16,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:16,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741882_1058 (size=12663) 2024-11-17T21:24:16,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741883_1059 (size=12663) 2024-11-17T21:24:16,718 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/60a59c50175e4c44a7db94bc527aadd5 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/60a59c50175e4c44a7db94bc527aadd5 2024-11-17T21:24:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T21:24:16,758 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/B of 8417a211937373f3c01713928c5cb18b into 60a59c50175e4c44a7db94bc527aadd5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:16,758 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:16,758 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:16,758 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/B, priority=13, startTime=1731878656577; duration=0sec 2024-11-17T21:24:16,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T21:24:16,759 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:16,759 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:B 2024-11-17T21:24:16,759 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:16,760 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:16,761 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/C is initiating minor compaction (all files) 2024-11-17T21:24:16,761 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/C in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:16,761 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d65710282e8b41b98c87f5f6c7006693, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1f08018227ce48ffbc6f0f191b29f9ad, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/08ae7c27a9db45a5a50562dc8bcfa83d] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=36.0 K 2024-11-17T21:24:16,763 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d65710282e8b41b98c87f5f6c7006693, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731878654380 2024-11-17T21:24:16,764 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f08018227ce48ffbc6f0f191b29f9ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731878654527 2024-11-17T21:24:16,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/dabed9892c3343be97c694cf9df6c02f is 50, key is test_row_0/A:col10/1731878656339/Put/seqid=0 2024-11-17T21:24:16,767 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 08ae7c27a9db45a5a50562dc8bcfa83d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731878655690 2024-11-17T21:24:16,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741884_1060 (size=12151) 2024-11-17T21:24:16,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/dabed9892c3343be97c694cf9df6c02f 2024-11-17T21:24:16,796 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#C#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:16,796 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/8790fc40da3c4ee0a8d8051602d415f8 is 50, key is test_row_0/C:col10/1731878655690/Put/seqid=0 2024-11-17T21:24:16,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/64e499bc9b3040ca989d3181e6db872a is 50, key is test_row_0/B:col10/1731878656339/Put/seqid=0 2024-11-17T21:24:16,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T21:24:16,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:16,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:16,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:16,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:16,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:16,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878716795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878716798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878716797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878716798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878716811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741885_1061 (size=12663) 2024-11-17T21:24:16,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741886_1062 (size=12151) 2024-11-17T21:24:16,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/64e499bc9b3040ca989d3181e6db872a 2024-11-17T21:24:16,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/4dbcba52061948b088390c408f3240b1 is 50, key is test_row_0/C:col10/1731878656339/Put/seqid=0 2024-11-17T21:24:16,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741887_1063 (size=12151) 2024-11-17T21:24:16,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/4dbcba52061948b088390c408f3240b1 2024-11-17T21:24:16,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/dabed9892c3343be97c694cf9df6c02f as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/dabed9892c3343be97c694cf9df6c02f 2024-11-17T21:24:16,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/dabed9892c3343be97c694cf9df6c02f, entries=150, sequenceid=234, filesize=11.9 K 2024-11-17T21:24:16,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/64e499bc9b3040ca989d3181e6db872a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/64e499bc9b3040ca989d3181e6db872a 2024-11-17T21:24:16,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/64e499bc9b3040ca989d3181e6db872a, entries=150, sequenceid=234, filesize=11.9 K 2024-11-17T21:24:16,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/4dbcba52061948b088390c408f3240b1 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/4dbcba52061948b088390c408f3240b1 2024-11-17T21:24:16,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/4dbcba52061948b088390c408f3240b1, entries=150, sequenceid=234, filesize=11.9 K 2024-11-17T21:24:16,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 8417a211937373f3c01713928c5cb18b in 153ms, sequenceid=234, compaction requested=false 2024-11-17T21:24:16,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:16,924 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:16,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T21:24:16,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/90878f5b18e54f0997343f20ae7b1e6d is 50, key is test_row_0/A:col10/1731878656793/Put/seqid=0 2024-11-17T21:24:16,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878716940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878716944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878716947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878716949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878716949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:16,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:16,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:16,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T21:24:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:16,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T21:24:16,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741888_1064 (size=14541) 2024-11-17T21:24:17,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878717061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:17,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:17,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878717061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:17,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:17,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878717061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:17,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878717061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:17,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878717062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:17,101 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/b2ae0225c58b4f84bcf5fe6b6ac95595 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b2ae0225c58b4f84bcf5fe6b6ac95595 2024-11-17T21:24:17,111 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/A of 8417a211937373f3c01713928c5cb18b into b2ae0225c58b4f84bcf5fe6b6ac95595(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:17,112 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:17,112 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/A, priority=13, startTime=1731878656577; duration=0sec 2024-11-17T21:24:17,112 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:17,112 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:A 2024-11-17T21:24:17,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T21:24:17,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:17,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:17,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:17,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,228 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/8790fc40da3c4ee0a8d8051602d415f8 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/8790fc40da3c4ee0a8d8051602d415f8 2024-11-17T21:24:17,239 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/C of 8417a211937373f3c01713928c5cb18b into 8790fc40da3c4ee0a8d8051602d415f8(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:17,239 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:17,239 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/C, priority=13, startTime=1731878656577; duration=0sec 2024-11-17T21:24:17,240 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:17,240 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:C 2024-11-17T21:24:17,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T21:24:17,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:17,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:17,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:17,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T21:24:17,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878717272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878717272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:17,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:17,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878717273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:17,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878717272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:17,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878717274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:17,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:17,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/90878f5b18e54f0997343f20ae7b1e6d 2024-11-17T21:24:17,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T21:24:17,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:17,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:17,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:17,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/ba825cfb48904d40bd8891e6916fc0d1 is 50, key is test_row_0/B:col10/1731878656793/Put/seqid=0 2024-11-17T21:24:17,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741889_1065 (size=12151) 2024-11-17T21:24:17,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T21:24:17,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:17,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:17,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:17,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878717581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878717581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:17,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878717581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:17,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:17,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878717581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:17,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:17,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878717591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:17,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T21:24:17,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:17,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:17,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:17,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T21:24:17,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/ba825cfb48904d40bd8891e6916fc0d1 2024-11-17T21:24:17,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/a9b38167405f495aa140432f622ef377 is 50, key is test_row_0/C:col10/1731878656793/Put/seqid=0 2024-11-17T21:24:17,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741890_1066 (size=12151) 2024-11-17T21:24:17,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T21:24:17,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:17,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:17,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:17,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:18,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T21:24:18,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:18,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:18,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:18,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:18,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:18,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878718091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:18,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:18,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878718091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:18,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878718093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:18,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878718101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:18,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878718102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:18,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T21:24:18,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:18,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:18,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:18,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:18,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:18,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/a9b38167405f495aa140432f622ef377 2024-11-17T21:24:18,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/90878f5b18e54f0997343f20ae7b1e6d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/90878f5b18e54f0997343f20ae7b1e6d 2024-11-17T21:24:18,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/90878f5b18e54f0997343f20ae7b1e6d, entries=200, sequenceid=255, filesize=14.2 K 2024-11-17T21:24:18,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/ba825cfb48904d40bd8891e6916fc0d1 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/ba825cfb48904d40bd8891e6916fc0d1 2024-11-17T21:24:18,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/ba825cfb48904d40bd8891e6916fc0d1, entries=150, sequenceid=255, filesize=11.9 K 2024-11-17T21:24:18,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/a9b38167405f495aa140432f622ef377 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a9b38167405f495aa140432f622ef377 2024-11-17T21:24:18,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a9b38167405f495aa140432f622ef377, entries=150, sequenceid=255, filesize=11.9 K 2024-11-17T21:24:18,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 8417a211937373f3c01713928c5cb18b in 1406ms, sequenceid=255, compaction requested=true 2024-11-17T21:24:18,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:18,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:18,330 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:18,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:18,330 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:18,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:18,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:18,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:18,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:18,332 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:18,332 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:18,332 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/A is initiating minor compaction (all files) 2024-11-17T21:24:18,332 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/B is initiating minor compaction (all files) 2024-11-17T21:24:18,332 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/A in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:18,332 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/B in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:18,332 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/60a59c50175e4c44a7db94bc527aadd5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/64e499bc9b3040ca989d3181e6db872a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/ba825cfb48904d40bd8891e6916fc0d1] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=36.1 K 2024-11-17T21:24:18,332 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b2ae0225c58b4f84bcf5fe6b6ac95595, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/dabed9892c3343be97c694cf9df6c02f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/90878f5b18e54f0997343f20ae7b1e6d] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=38.4 K 2024-11-17T21:24:18,333 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2ae0225c58b4f84bcf5fe6b6ac95595, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731878655690 2024-11-17T21:24:18,333 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 60a59c50175e4c44a7db94bc527aadd5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731878655690 2024-11-17T21:24:18,336 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting dabed9892c3343be97c694cf9df6c02f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731878656339 2024-11-17T21:24:18,336 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 64e499bc9b3040ca989d3181e6db872a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731878656339 2024-11-17T21:24:18,336 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90878f5b18e54f0997343f20ae7b1e6d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731878656791 2024-11-17T21:24:18,337 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting ba825cfb48904d40bd8891e6916fc0d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731878656791 2024-11-17T21:24:18,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T21:24:18,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:18,351 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T21:24:18,361 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#A#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:18,361 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/355fdda0fef8456b8ac5d90b3d46fd8b is 50, key is test_row_0/A:col10/1731878656793/Put/seqid=0 2024-11-17T21:24:18,366 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#B#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:18,367 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/c8c16adaa19145fb97569c53ceca90f3 is 50, key is test_row_0/B:col10/1731878656793/Put/seqid=0 2024-11-17T21:24:18,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/9b557f31e586441bb6202f1db5977c51 is 50, key is test_row_0/A:col10/1731878656931/Put/seqid=0 2024-11-17T21:24:18,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741892_1068 (size=12765) 2024-11-17T21:24:18,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741891_1067 (size=12765) 2024-11-17T21:24:18,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741893_1069 (size=12301) 2024-11-17T21:24:18,436 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/c8c16adaa19145fb97569c53ceca90f3 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/c8c16adaa19145fb97569c53ceca90f3 2024-11-17T21:24:18,436 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/9b557f31e586441bb6202f1db5977c51 2024-11-17T21:24:18,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/e04112f3dde24c63ad8ea14a91de5208 is 50, key is test_row_0/B:col10/1731878656931/Put/seqid=0 2024-11-17T21:24:18,454 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/B of 8417a211937373f3c01713928c5cb18b into c8c16adaa19145fb97569c53ceca90f3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:18,454 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:18,454 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/B, priority=13, startTime=1731878658330; duration=0sec 2024-11-17T21:24:18,454 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:18,454 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:B 2024-11-17T21:24:18,454 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:18,457 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:18,457 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/C is initiating minor compaction (all files) 2024-11-17T21:24:18,457 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/C in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:18,458 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/8790fc40da3c4ee0a8d8051602d415f8, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/4dbcba52061948b088390c408f3240b1, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a9b38167405f495aa140432f622ef377] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=36.1 K 2024-11-17T21:24:18,458 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 8790fc40da3c4ee0a8d8051602d415f8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731878655690 2024-11-17T21:24:18,459 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dbcba52061948b088390c408f3240b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731878656339 2024-11-17T21:24:18,460 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting a9b38167405f495aa140432f622ef377, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731878656791 2024-11-17T21:24:18,489 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#C#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:18,490 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/3c365229a8e64b6eb841ea2d9faff8ef is 50, key is test_row_0/C:col10/1731878656793/Put/seqid=0 2024-11-17T21:24:18,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741894_1070 (size=12301) 2024-11-17T21:24:18,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741895_1071 (size=12765) 2024-11-17T21:24:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T21:24:18,834 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/355fdda0fef8456b8ac5d90b3d46fd8b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/355fdda0fef8456b8ac5d90b3d46fd8b 2024-11-17T21:24:18,843 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/A of 8417a211937373f3c01713928c5cb18b into 355fdda0fef8456b8ac5d90b3d46fd8b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:18,843 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:18,843 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/A, priority=13, startTime=1731878658330; duration=0sec 2024-11-17T21:24:18,843 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:18,843 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:A 2024-11-17T21:24:18,892 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/e04112f3dde24c63ad8ea14a91de5208 2024-11-17T21:24:18,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/1fb7f9175fe34de6a535b02ab235b666 is 50, key is test_row_0/C:col10/1731878656931/Put/seqid=0 2024-11-17T21:24:18,917 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/3c365229a8e64b6eb841ea2d9faff8ef as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/3c365229a8e64b6eb841ea2d9faff8ef 2024-11-17T21:24:18,925 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/C of 8417a211937373f3c01713928c5cb18b into 3c365229a8e64b6eb841ea2d9faff8ef(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:18,925 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:18,925 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/C, priority=13, startTime=1731878658331; duration=0sec 2024-11-17T21:24:18,925 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:18,925 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:C 2024-11-17T21:24:18,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741896_1072 (size=12301) 2024-11-17T21:24:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:19,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:19,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878719135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878719136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878719138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878719140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878719143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878719251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878719251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878719252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878719252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878719252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,334 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/1fb7f9175fe34de6a535b02ab235b666 2024-11-17T21:24:19,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/9b557f31e586441bb6202f1db5977c51 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/9b557f31e586441bb6202f1db5977c51 2024-11-17T21:24:19,352 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/9b557f31e586441bb6202f1db5977c51, entries=150, sequenceid=272, filesize=12.0 K 2024-11-17T21:24:19,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/e04112f3dde24c63ad8ea14a91de5208 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/e04112f3dde24c63ad8ea14a91de5208 2024-11-17T21:24:19,361 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/e04112f3dde24c63ad8ea14a91de5208, entries=150, sequenceid=272, filesize=12.0 K 2024-11-17T21:24:19,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/1fb7f9175fe34de6a535b02ab235b666 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1fb7f9175fe34de6a535b02ab235b666 2024-11-17T21:24:19,374 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1fb7f9175fe34de6a535b02ab235b666, entries=150, sequenceid=272, filesize=12.0 K 2024-11-17T21:24:19,377 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 8417a211937373f3c01713928c5cb18b in 1025ms, sequenceid=272, compaction requested=false 2024-11-17T21:24:19,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:19,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:19,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-17T21:24:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-17T21:24:19,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-17T21:24:19,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7290 sec 2024-11-17T21:24:19,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.7400 sec 2024-11-17T21:24:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:19,464 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:19,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T21:24:19,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/37e861997d2e45d3854e92f2c7c60578 is 50, key is test_row_0/A:col10/1731878659464/Put/seqid=0 2024-11-17T21:24:19,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878719479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878719481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878719481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878719485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878719487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741897_1073 (size=12301) 2024-11-17T21:24:19,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/37e861997d2e45d3854e92f2c7c60578 2024-11-17T21:24:19,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/9eeb7669b7594513babd4f477974b783 is 50, key is test_row_0/B:col10/1731878659464/Put/seqid=0 2024-11-17T21:24:19,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741898_1074 (size=12301) 2024-11-17T21:24:19,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/9eeb7669b7594513babd4f477974b783 2024-11-17T21:24:19,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/c23be52488814173853db70a92f22d16 is 50, key is test_row_0/C:col10/1731878659464/Put/seqid=0 2024-11-17T21:24:19,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741899_1075 (size=12301) 2024-11-17T21:24:19,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/c23be52488814173853db70a92f22d16 2024-11-17T21:24:19,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/37e861997d2e45d3854e92f2c7c60578 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/37e861997d2e45d3854e92f2c7c60578 2024-11-17T21:24:19,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/37e861997d2e45d3854e92f2c7c60578, entries=150, sequenceid=299, filesize=12.0 K 2024-11-17T21:24:19,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/9eeb7669b7594513babd4f477974b783 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/9eeb7669b7594513babd4f477974b783 2024-11-17T21:24:19,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/9eeb7669b7594513babd4f477974b783, entries=150, sequenceid=299, filesize=12.0 K 2024-11-17T21:24:19,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/c23be52488814173853db70a92f22d16 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c23be52488814173853db70a92f22d16 2024-11-17T21:24:19,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878719591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878719592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878719592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c23be52488814173853db70a92f22d16, entries=150, sequenceid=299, filesize=12.0 K 2024-11-17T21:24:19,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878719591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 8417a211937373f3c01713928c5cb18b in 132ms, sequenceid=299, compaction requested=true 2024-11-17T21:24:19,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:19,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:19,596 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:19,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:19,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:19,597 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:19,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:19,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:19,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:19,597 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:19,597 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:19,598 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/A is initiating minor compaction (all files) 2024-11-17T21:24:19,598 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/B is initiating minor compaction (all files) 2024-11-17T21:24:19,598 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/A in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:19,598 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/B in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:19,598 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/355fdda0fef8456b8ac5d90b3d46fd8b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/9b557f31e586441bb6202f1db5977c51, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/37e861997d2e45d3854e92f2c7c60578] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=36.5 K 2024-11-17T21:24:19,598 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 355fdda0fef8456b8ac5d90b3d46fd8b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731878656791 2024-11-17T21:24:19,599 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b557f31e586441bb6202f1db5977c51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731878656931 2024-11-17T21:24:19,599 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/c8c16adaa19145fb97569c53ceca90f3, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/e04112f3dde24c63ad8ea14a91de5208, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/9eeb7669b7594513babd4f477974b783] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=36.5 K 2024-11-17T21:24:19,599 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37e861997d2e45d3854e92f2c7c60578, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731878659140 2024-11-17T21:24:19,599 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting c8c16adaa19145fb97569c53ceca90f3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731878656791 2024-11-17T21:24:19,600 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting e04112f3dde24c63ad8ea14a91de5208, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731878656931 2024-11-17T21:24:19,600 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 9eeb7669b7594513babd4f477974b783, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731878659140 2024-11-17T21:24:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:19,605 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:19,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T21:24:19,614 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:19,615 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/bc3ff480c1a64e67a0a7d8af66b6fc4c is 50, key is test_row_0/B:col10/1731878659464/Put/seqid=0 2024-11-17T21:24:19,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/40056f446fc94af3adecb3959b9cc951 is 50, key is test_row_0/A:col10/1731878659603/Put/seqid=0 2024-11-17T21:24:19,632 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#A#compaction#65 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:19,633 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/696fee97238048c3b880d9a1b86b1dca is 50, key is test_row_0/A:col10/1731878659464/Put/seqid=0 2024-11-17T21:24:19,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741900_1076 (size=13017) 2024-11-17T21:24:19,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741901_1077 (size=14741) 2024-11-17T21:24:19,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741902_1078 (size=13017) 2024-11-17T21:24:19,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878719705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878719801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878719801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878719802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878719802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:19,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:19,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878719811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878720021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,045 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/bc3ff480c1a64e67a0a7d8af66b6fc4c as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/bc3ff480c1a64e67a0a7d8af66b6fc4c 2024-11-17T21:24:20,052 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/B of 8417a211937373f3c01713928c5cb18b into bc3ff480c1a64e67a0a7d8af66b6fc4c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:20,052 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:20,052 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/B, priority=13, startTime=1731878659596; duration=0sec 2024-11-17T21:24:20,052 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:20,052 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:B 2024-11-17T21:24:20,052 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:20,054 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:20,054 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/C is initiating minor compaction (all files) 2024-11-17T21:24:20,054 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/C in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:20,054 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/3c365229a8e64b6eb841ea2d9faff8ef, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1fb7f9175fe34de6a535b02ab235b666, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c23be52488814173853db70a92f22d16] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=36.5 K 2024-11-17T21:24:20,054 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c365229a8e64b6eb841ea2d9faff8ef, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731878656791 2024-11-17T21:24:20,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/40056f446fc94af3adecb3959b9cc951 2024-11-17T21:24:20,055 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fb7f9175fe34de6a535b02ab235b666, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731878656931 2024-11-17T21:24:20,055 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting c23be52488814173853db70a92f22d16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731878659140 2024-11-17T21:24:20,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/38117ac7868643e58cb83b05efbc5d89 is 50, key is test_row_0/B:col10/1731878659603/Put/seqid=0 2024-11-17T21:24:20,069 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#C#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:20,069 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/a56a3393798c4214ba45954b22ae7197 is 50, key is test_row_0/C:col10/1731878659464/Put/seqid=0 2024-11-17T21:24:20,075 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/696fee97238048c3b880d9a1b86b1dca as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/696fee97238048c3b880d9a1b86b1dca 2024-11-17T21:24:20,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741903_1079 (size=13017) 2024-11-17T21:24:20,087 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/a56a3393798c4214ba45954b22ae7197 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a56a3393798c4214ba45954b22ae7197 2024-11-17T21:24:20,089 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/A of 8417a211937373f3c01713928c5cb18b into 696fee97238048c3b880d9a1b86b1dca(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:20,089 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:20,089 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/A, priority=13, startTime=1731878659596; duration=0sec 2024-11-17T21:24:20,089 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:20,089 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:A 2024-11-17T21:24:20,095 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/C of 8417a211937373f3c01713928c5cb18b into a56a3393798c4214ba45954b22ae7197(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:20,095 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:20,095 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/C, priority=13, startTime=1731878659597; duration=0sec 2024-11-17T21:24:20,096 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:20,096 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:C 2024-11-17T21:24:20,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741904_1080 (size=12301) 2024-11-17T21:24:20,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878720111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878720112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878720112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878720112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878720331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/38117ac7868643e58cb83b05efbc5d89 2024-11-17T21:24:20,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/d967f90cfec344b4b894cd6bda2e4d10 is 50, key is test_row_0/C:col10/1731878659603/Put/seqid=0 2024-11-17T21:24:20,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741905_1081 (size=12301) 2024-11-17T21:24:20,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/d967f90cfec344b4b894cd6bda2e4d10 2024-11-17T21:24:20,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/40056f446fc94af3adecb3959b9cc951 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/40056f446fc94af3adecb3959b9cc951 2024-11-17T21:24:20,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/40056f446fc94af3adecb3959b9cc951, entries=200, sequenceid=313, filesize=14.4 K 2024-11-17T21:24:20,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/38117ac7868643e58cb83b05efbc5d89 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/38117ac7868643e58cb83b05efbc5d89 2024-11-17T21:24:20,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/38117ac7868643e58cb83b05efbc5d89, entries=150, sequenceid=313, filesize=12.0 K 2024-11-17T21:24:20,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/d967f90cfec344b4b894cd6bda2e4d10 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d967f90cfec344b4b894cd6bda2e4d10 2024-11-17T21:24:20,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d967f90cfec344b4b894cd6bda2e4d10, entries=150, sequenceid=313, filesize=12.0 K 2024-11-17T21:24:20,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 8417a211937373f3c01713928c5cb18b in 949ms, sequenceid=313, compaction requested=false 2024-11-17T21:24:20,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:20,624 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:20,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T21:24:20,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/6885f522193c4ed0b228fed312ad4082 is 50, key is test_row_0/A:col10/1731878660623/Put/seqid=0 2024-11-17T21:24:20,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878720635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878720638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878720639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878720649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741906_1082 (size=14741) 2024-11-17T21:24:20,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/6885f522193c4ed0b228fed312ad4082 2024-11-17T21:24:20,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/7be046f6ea774456a59c2bedfd2ac237 is 50, key is test_row_0/B:col10/1731878660623/Put/seqid=0 2024-11-17T21:24:20,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741907_1083 (size=12301) 2024-11-17T21:24:20,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/7be046f6ea774456a59c2bedfd2ac237 2024-11-17T21:24:20,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/c6035e45dbbb44369b988633add2a5b9 is 50, key is test_row_0/C:col10/1731878660623/Put/seqid=0 2024-11-17T21:24:20,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741908_1084 (size=12301) 2024-11-17T21:24:20,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/c6035e45dbbb44369b988633add2a5b9 2024-11-17T21:24:20,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/6885f522193c4ed0b228fed312ad4082 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6885f522193c4ed0b228fed312ad4082 2024-11-17T21:24:20,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6885f522193c4ed0b228fed312ad4082, entries=200, sequenceid=339, filesize=14.4 K 2024-11-17T21:24:20,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/7be046f6ea774456a59c2bedfd2ac237 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/7be046f6ea774456a59c2bedfd2ac237 2024-11-17T21:24:20,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/7be046f6ea774456a59c2bedfd2ac237, entries=150, sequenceid=339, filesize=12.0 K 2024-11-17T21:24:20,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/c6035e45dbbb44369b988633add2a5b9 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c6035e45dbbb44369b988633add2a5b9 2024-11-17T21:24:20,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c6035e45dbbb44369b988633add2a5b9, entries=150, sequenceid=339, filesize=12.0 K 2024-11-17T21:24:20,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 8417a211937373f3c01713928c5cb18b in 127ms, sequenceid=339, compaction requested=true 2024-11-17T21:24:20,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:20,752 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:20,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:20,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:20,752 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:20,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:20,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:20,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:20,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:20,753 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:20,754 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/A is initiating minor compaction (all files) 2024-11-17T21:24:20,754 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/A in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:20,754 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/696fee97238048c3b880d9a1b86b1dca, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/40056f446fc94af3adecb3959b9cc951, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6885f522193c4ed0b228fed312ad4082] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=41.5 K 2024-11-17T21:24:20,754 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:20,754 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/B is initiating minor compaction (all files) 2024-11-17T21:24:20,754 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/B in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:20,754 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/bc3ff480c1a64e67a0a7d8af66b6fc4c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/38117ac7868643e58cb83b05efbc5d89, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/7be046f6ea774456a59c2bedfd2ac237] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=36.7 K 2024-11-17T21:24:20,755 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 696fee97238048c3b880d9a1b86b1dca, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731878659140 2024-11-17T21:24:20,755 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting bc3ff480c1a64e67a0a7d8af66b6fc4c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731878659140 2024-11-17T21:24:20,755 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40056f446fc94af3adecb3959b9cc951, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731878659478 2024-11-17T21:24:20,756 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 38117ac7868643e58cb83b05efbc5d89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731878659483 2024-11-17T21:24:20,756 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6885f522193c4ed0b228fed312ad4082, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731878659659 2024-11-17T21:24:20,759 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:20,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T21:24:20,759 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 7be046f6ea774456a59c2bedfd2ac237, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731878659659 2024-11-17T21:24:20,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:20,774 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:20,774 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#A#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:20,775 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/36dd30778ca74d13a8dad620c2502ef2 is 50, key is test_row_0/B:col10/1731878660623/Put/seqid=0 2024-11-17T21:24:20,775 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/08a67300a9954155adfd8b03225f2e29 is 50, key is test_row_0/A:col10/1731878660623/Put/seqid=0 2024-11-17T21:24:20,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/79f28ad6193e4a39ae8109657b75be92 is 50, key is test_row_0/A:col10/1731878660756/Put/seqid=0 2024-11-17T21:24:20,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741910_1086 (size=13119) 2024-11-17T21:24:20,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878720806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878720807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T21:24:20,811 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:20,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:20,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-17T21:24:20,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878720808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878720809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T21:24:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,816 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:20,819 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:20,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:20,823 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/08a67300a9954155adfd8b03225f2e29 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/08a67300a9954155adfd8b03225f2e29 2024-11-17T21:24:20,833 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/A of 8417a211937373f3c01713928c5cb18b into 08a67300a9954155adfd8b03225f2e29(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:20,833 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:20,833 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/A, priority=13, startTime=1731878660752; duration=0sec 2024-11-17T21:24:20,833 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:20,833 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:A 2024-11-17T21:24:20,833 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:20,834 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:20,834 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/C is initiating minor compaction (all files) 2024-11-17T21:24:20,834 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/C in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:20,834 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a56a3393798c4214ba45954b22ae7197, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d967f90cfec344b4b894cd6bda2e4d10, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c6035e45dbbb44369b988633add2a5b9] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=36.7 K 2024-11-17T21:24:20,835 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting a56a3393798c4214ba45954b22ae7197, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731878659140 2024-11-17T21:24:20,835 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting d967f90cfec344b4b894cd6bda2e4d10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731878659483 2024-11-17T21:24:20,835 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6035e45dbbb44369b988633add2a5b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731878659659 2024-11-17T21:24:20,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741909_1085 (size=13119) 2024-11-17T21:24:20,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741911_1087 (size=14741) 2024-11-17T21:24:20,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/79f28ad6193e4a39ae8109657b75be92 2024-11-17T21:24:20,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878720841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,850 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/36dd30778ca74d13a8dad620c2502ef2 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/36dd30778ca74d13a8dad620c2502ef2 2024-11-17T21:24:20,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/f44ea9d05b6d4ee09129649aa3502246 is 50, key is test_row_0/B:col10/1731878660756/Put/seqid=0 2024-11-17T21:24:20,863 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/B of 8417a211937373f3c01713928c5cb18b into 36dd30778ca74d13a8dad620c2502ef2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:20,863 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:20,863 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/B, priority=13, startTime=1731878660752; duration=0sec 2024-11-17T21:24:20,863 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:20,863 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:B 2024-11-17T21:24:20,870 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#C#compaction#76 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:20,871 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/d44255eea0c346f9997810985e3e9e00 is 50, key is test_row_0/C:col10/1731878660623/Put/seqid=0 2024-11-17T21:24:20,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741912_1088 (size=12301) 2024-11-17T21:24:20,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741913_1089 (size=13119) 2024-11-17T21:24:20,888 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/d44255eea0c346f9997810985e3e9e00 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d44255eea0c346f9997810985e3e9e00 2024-11-17T21:24:20,900 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/C of 8417a211937373f3c01713928c5cb18b into d44255eea0c346f9997810985e3e9e00(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:20,900 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:20,900 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/C, priority=13, startTime=1731878660753; duration=0sec 2024-11-17T21:24:20,900 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:20,900 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:C 2024-11-17T21:24:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T21:24:20,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878720921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878720921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878720921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878720922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:20,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T21:24:20,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:20,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:20,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:20,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:20,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:21,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T21:24:21,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:21,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:21,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:21,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:21,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T21:24:21,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878721132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878721132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878721132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878721132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/f44ea9d05b6d4ee09129649aa3502246 2024-11-17T21:24:21,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T21:24:21,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:21,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:21,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:21,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:21,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:21,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/6c76b115aa454604a1c9b92af0aa915c is 50, key is test_row_0/C:col10/1731878660756/Put/seqid=0 2024-11-17T21:24:21,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741914_1090 (size=12301) 2024-11-17T21:24:21,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/6c76b115aa454604a1c9b92af0aa915c 2024-11-17T21:24:21,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/79f28ad6193e4a39ae8109657b75be92 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/79f28ad6193e4a39ae8109657b75be92 2024-11-17T21:24:21,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/79f28ad6193e4a39ae8109657b75be92, entries=200, sequenceid=355, filesize=14.4 K 2024-11-17T21:24:21,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/f44ea9d05b6d4ee09129649aa3502246 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/f44ea9d05b6d4ee09129649aa3502246 2024-11-17T21:24:21,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/f44ea9d05b6d4ee09129649aa3502246, entries=150, sequenceid=355, filesize=12.0 K 2024-11-17T21:24:21,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/6c76b115aa454604a1c9b92af0aa915c as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/6c76b115aa454604a1c9b92af0aa915c 2024-11-17T21:24:21,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/6c76b115aa454604a1c9b92af0aa915c, entries=150, sequenceid=355, filesize=12.0 K 2024-11-17T21:24:21,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 8417a211937373f3c01713928c5cb18b in 574ms, sequenceid=355, compaction requested=false 2024-11-17T21:24:21,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T21:24:21,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T21:24:21,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:21,443 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T21:24:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:21,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:21,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/b16ac52a46d9489fbed907053a02deab is 50, key is test_row_0/A:col10/1731878660803/Put/seqid=0 2024-11-17T21:24:21,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741915_1091 (size=12301) 2024-11-17T21:24:21,468 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/b16ac52a46d9489fbed907053a02deab 2024-11-17T21:24:21,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/4713356991cc46d69222d53a7546743d is 50, key is test_row_0/B:col10/1731878660803/Put/seqid=0 2024-11-17T21:24:21,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741916_1092 (size=12301) 2024-11-17T21:24:21,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878721501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878721506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878721506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878721510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878721611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878721611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878721611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878721631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878721821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878721821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878721822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878721841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878721861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:21,867 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_2' in 8417a211937373f3c01713928c5cb18b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:24:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:21,883 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/4713356991cc46d69222d53a7546743d 2024-11-17T21:24:21,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/44b3ed94c0b54c878a20e4d03a5b2f5b is 50, key is test_row_0/C:col10/1731878660803/Put/seqid=0 2024-11-17T21:24:21,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741917_1093 (size=12301) 2024-11-17T21:24:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T21:24:22,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878722131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878722131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878722131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:22,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878722151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:22,299 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/44b3ed94c0b54c878a20e4d03a5b2f5b 2024-11-17T21:24:22,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/b16ac52a46d9489fbed907053a02deab as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b16ac52a46d9489fbed907053a02deab 2024-11-17T21:24:22,312 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b16ac52a46d9489fbed907053a02deab, entries=150, sequenceid=379, filesize=12.0 K 2024-11-17T21:24:22,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/4713356991cc46d69222d53a7546743d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/4713356991cc46d69222d53a7546743d 2024-11-17T21:24:22,322 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/4713356991cc46d69222d53a7546743d, entries=150, sequenceid=379, filesize=12.0 K 2024-11-17T21:24:22,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/44b3ed94c0b54c878a20e4d03a5b2f5b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/44b3ed94c0b54c878a20e4d03a5b2f5b 2024-11-17T21:24:22,331 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/44b3ed94c0b54c878a20e4d03a5b2f5b, entries=150, sequenceid=379, filesize=12.0 K 2024-11-17T21:24:22,332 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 8417a211937373f3c01713928c5cb18b in 889ms, sequenceid=379, compaction requested=true 2024-11-17T21:24:22,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:22,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:22,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-17T21:24:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-17T21:24:22,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-17T21:24:22,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5150 sec 2024-11-17T21:24:22,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.5240 sec 2024-11-17T21:24:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:22,643 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:22,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-17T21:24:22,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878722663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:22,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878722666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:22,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:22,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878722667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:22,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:22,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878722667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:22,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:22,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:22,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/3f3b1624f9054b3e8c168d117746ba72 is 50, key is test_row_0/A:col10/1731878662643/Put/seqid=0 2024-11-17T21:24:22,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741918_1094 (size=12301) 2024-11-17T21:24:22,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/3f3b1624f9054b3e8c168d117746ba72 2024-11-17T21:24:22,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/0015c2646e8c4c79b403d994b305c4dd is 50, key is test_row_0/B:col10/1731878662643/Put/seqid=0 2024-11-17T21:24:22,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741919_1095 (size=12301) 2024-11-17T21:24:22,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878722771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878722771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:22,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878722780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:22,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878722781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T21:24:22,961 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:22,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-17T21:24:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T21:24:22,964 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:22,965 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:22,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:22,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878722980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:22,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878722981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:22,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878722991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:22,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878722991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T21:24:23,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/0015c2646e8c4c79b403d994b305c4dd 2024-11-17T21:24:23,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T21:24:23,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:23,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:23,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:23,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:23,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:23,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/5b204c639d3043dfa53df4f00e81c27c is 50, key is test_row_0/C:col10/1731878662643/Put/seqid=0 2024-11-17T21:24:23,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741920_1096 (size=12301) 2024-11-17T21:24:23,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T21:24:23,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:23,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:23,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:23,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:23,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T21:24:23,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:23,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878723291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878723291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:23,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:23,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878723301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878723301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:23,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T21:24:23,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:23,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:23,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:23,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:23,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:23,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/5b204c639d3043dfa53df4f00e81c27c 2024-11-17T21:24:23,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/3f3b1624f9054b3e8c168d117746ba72 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/3f3b1624f9054b3e8c168d117746ba72 2024-11-17T21:24:23,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/3f3b1624f9054b3e8c168d117746ba72, entries=150, sequenceid=397, filesize=12.0 K 2024-11-17T21:24:23,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/0015c2646e8c4c79b403d994b305c4dd as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0015c2646e8c4c79b403d994b305c4dd 2024-11-17T21:24:23,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0015c2646e8c4c79b403d994b305c4dd, entries=150, sequenceid=397, filesize=12.0 K 2024-11-17T21:24:23,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/5b204c639d3043dfa53df4f00e81c27c as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/5b204c639d3043dfa53df4f00e81c27c 2024-11-17T21:24:23,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/5b204c639d3043dfa53df4f00e81c27c, entries=150, sequenceid=397, filesize=12.0 K 2024-11-17T21:24:23,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 8417a211937373f3c01713928c5cb18b in 914ms, sequenceid=397, compaction requested=true 2024-11-17T21:24:23,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:23,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:23,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:23,558 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:24:23,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:23,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:23,558 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:24:23,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:23,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:23,559 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52462 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:24:23,559 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/A is initiating minor compaction (all files) 2024-11-17T21:24:23,559 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/A in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:23,560 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/08a67300a9954155adfd8b03225f2e29, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/79f28ad6193e4a39ae8109657b75be92, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b16ac52a46d9489fbed907053a02deab, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/3f3b1624f9054b3e8c168d117746ba72] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=51.2 K 2024-11-17T21:24:23,560 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:24:23,560 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/B is initiating minor compaction (all files) 2024-11-17T21:24:23,560 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/B in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:23,560 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/36dd30778ca74d13a8dad620c2502ef2, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/f44ea9d05b6d4ee09129649aa3502246, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/4713356991cc46d69222d53a7546743d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0015c2646e8c4c79b403d994b305c4dd] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=48.8 K 2024-11-17T21:24:23,560 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08a67300a9954155adfd8b03225f2e29, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731878659659 2024-11-17T21:24:23,561 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 36dd30778ca74d13a8dad620c2502ef2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731878659659 2024-11-17T21:24:23,561 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting f44ea9d05b6d4ee09129649aa3502246, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731878660629 2024-11-17T21:24:23,561 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79f28ad6193e4a39ae8109657b75be92, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731878660629 2024-11-17T21:24:23,562 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 4713356991cc46d69222d53a7546743d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731878660803 2024-11-17T21:24:23,562 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting b16ac52a46d9489fbed907053a02deab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731878660803 2024-11-17T21:24:23,562 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 0015c2646e8c4c79b403d994b305c4dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731878661472 2024-11-17T21:24:23,563 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f3b1624f9054b3e8c168d117746ba72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731878661472 2024-11-17T21:24:23,577 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#B#compaction#84 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:23,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T21:24:23,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:23,577 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-17T21:24:23,578 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/bf6ce90f88b14f0aad0f3478a2e56c67 is 50, key is test_row_0/B:col10/1731878662643/Put/seqid=0 2024-11-17T21:24:23,587 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#A#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:23,588 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/d34dfc501dd44533a63bf9a5a6626367 is 50, key is test_row_0/A:col10/1731878662643/Put/seqid=0 2024-11-17T21:24:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T21:24:23,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/08889fef91e849ce8e7171f202ed230f is 50, key is test_row_0/A:col10/1731878662666/Put/seqid=0 2024-11-17T21:24:23,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741921_1097 (size=13255) 2024-11-17T21:24:23,617 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/bf6ce90f88b14f0aad0f3478a2e56c67 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/bf6ce90f88b14f0aad0f3478a2e56c67 2024-11-17T21:24:23,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741922_1098 (size=13255) 2024-11-17T21:24:23,625 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8417a211937373f3c01713928c5cb18b/B of 8417a211937373f3c01713928c5cb18b into bf6ce90f88b14f0aad0f3478a2e56c67(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:23,626 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:23,626 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/B, priority=12, startTime=1731878663558; duration=0sec 2024-11-17T21:24:23,630 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:23,630 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:B 2024-11-17T21:24:23,630 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:24:23,630 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/d34dfc501dd44533a63bf9a5a6626367 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/d34dfc501dd44533a63bf9a5a6626367 2024-11-17T21:24:23,631 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:24:23,634 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/C is initiating minor compaction (all files) 2024-11-17T21:24:23,635 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/C in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:23,635 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d44255eea0c346f9997810985e3e9e00, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/6c76b115aa454604a1c9b92af0aa915c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/44b3ed94c0b54c878a20e4d03a5b2f5b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/5b204c639d3043dfa53df4f00e81c27c] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=48.8 K 2024-11-17T21:24:23,636 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d44255eea0c346f9997810985e3e9e00, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731878659659 2024-11-17T21:24:23,637 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c76b115aa454604a1c9b92af0aa915c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731878660629 2024-11-17T21:24:23,637 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 44b3ed94c0b54c878a20e4d03a5b2f5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731878660803 2024-11-17T21:24:23,637 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b204c639d3043dfa53df4f00e81c27c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731878661472 2024-11-17T21:24:23,639 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8417a211937373f3c01713928c5cb18b/A of 8417a211937373f3c01713928c5cb18b into d34dfc501dd44533a63bf9a5a6626367(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:23,639 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:23,639 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/A, priority=12, startTime=1731878663557; duration=0sec 2024-11-17T21:24:23,639 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:23,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741923_1099 (size=12301) 2024-11-17T21:24:23,639 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:A 2024-11-17T21:24:23,642 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/08889fef91e849ce8e7171f202ed230f 2024-11-17T21:24:23,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/0b19789033674b4193cabf84a51cfb3e is 50, key is test_row_0/B:col10/1731878662666/Put/seqid=0 2024-11-17T21:24:23,662 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#C#compaction#88 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:23,662 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/84cb72b76e6242bc83bef2fc21247ce6 is 50, key is test_row_0/C:col10/1731878662643/Put/seqid=0 2024-11-17T21:24:23,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741925_1101 (size=13255) 2024-11-17T21:24:23,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741924_1100 (size=12301) 2024-11-17T21:24:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:23,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:23,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878723822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:23,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878723823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:23,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878723824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:23,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878723826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:23,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878723880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:23,885 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_2' in 8417a211937373f3c01713928c5cb18b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:24:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:23,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878723932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878723932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:23,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878723932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:23,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:23,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878723941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:24,084 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/0b19789033674b4193cabf84a51cfb3e 2024-11-17T21:24:24,090 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/84cb72b76e6242bc83bef2fc21247ce6 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/84cb72b76e6242bc83bef2fc21247ce6 2024-11-17T21:24:24,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/40c792ed46614c62acbc69fe3f453f99 is 50, key is test_row_0/C:col10/1731878662666/Put/seqid=0 2024-11-17T21:24:24,098 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8417a211937373f3c01713928c5cb18b/C of 8417a211937373f3c01713928c5cb18b into 84cb72b76e6242bc83bef2fc21247ce6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:24,098 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:24,098 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/C, priority=12, startTime=1731878663558; duration=0sec 2024-11-17T21:24:24,098 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:24,098 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:C 2024-11-17T21:24:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T21:24:24,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741926_1102 (size=12301) 2024-11-17T21:24:24,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878724141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878724141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878724142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:24,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878724151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:24,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:24,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878724451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:24,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878724452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:24,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878724452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:24,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878724461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:24,504 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/40c792ed46614c62acbc69fe3f453f99 2024-11-17T21:24:24,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/08889fef91e849ce8e7171f202ed230f as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/08889fef91e849ce8e7171f202ed230f 2024-11-17T21:24:24,524 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/08889fef91e849ce8e7171f202ed230f, entries=150, sequenceid=415, filesize=12.0 K 2024-11-17T21:24:24,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/0b19789033674b4193cabf84a51cfb3e as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0b19789033674b4193cabf84a51cfb3e 2024-11-17T21:24:24,532 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0b19789033674b4193cabf84a51cfb3e, entries=150, sequenceid=415, filesize=12.0 K 2024-11-17T21:24:24,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/40c792ed46614c62acbc69fe3f453f99 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/40c792ed46614c62acbc69fe3f453f99 2024-11-17T21:24:24,539 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/40c792ed46614c62acbc69fe3f453f99, entries=150, sequenceid=415, filesize=12.0 K 2024-11-17T21:24:24,540 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 8417a211937373f3c01713928c5cb18b in 963ms, sequenceid=415, compaction requested=false 2024-11-17T21:24:24,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:24,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:24,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-17T21:24:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-17T21:24:24,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-17T21:24:24,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5760 sec 2024-11-17T21:24:24,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.5820 sec 2024-11-17T21:24:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:24,963 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:24,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-17T21:24:24,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/c8ba42d3b18349a187ab26f1a94f36af is 50, key is test_row_0/A:col10/1731878664962/Put/seqid=0 2024-11-17T21:24:24,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741927_1103 (size=14741) 2024-11-17T21:24:24,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878724980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:24,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:24,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878724985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:24,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878724986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:24,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878724987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:25,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:25,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878725091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878725091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:25,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:25,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878725101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878725101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T21:24:25,111 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:25,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-17T21:24:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T21:24:25,113 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:25,114 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:25,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:25,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T21:24:25,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T21:24:25,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:25,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:25,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:25,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:25,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:25,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:25,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878725301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:25,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878725301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:25,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:25,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878725311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:25,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:25,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878725311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:25,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:25,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:25,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/c8ba42d3b18349a187ab26f1a94f36af 2024-11-17T21:24:25,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/70814d2353ce407bacd1cd2171edc9af is 50, key is test_row_0/B:col10/1731878664962/Put/seqid=0 2024-11-17T21:24:25,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741928_1104 (size=12301) 2024-11-17T21:24:25,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T21:24:25,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:25,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:25,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:25,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:25,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T21:24:25,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T21:24:25,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:25,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:25,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:25,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:25,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:25,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878725611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:25,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878725611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:25,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:25,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878725621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:25,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878725621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:25,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T21:24:25,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:25,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:25,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:25,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:25,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T21:24:25,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/70814d2353ce407bacd1cd2171edc9af 2024-11-17T21:24:25,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/68f48769b71f4d53acc1b94d89321c3d is 50, key is test_row_0/C:col10/1731878664962/Put/seqid=0 2024-11-17T21:24:25,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741929_1105 (size=12301) 2024-11-17T21:24:25,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/68f48769b71f4d53acc1b94d89321c3d 2024-11-17T21:24:25,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/c8ba42d3b18349a187ab26f1a94f36af as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/c8ba42d3b18349a187ab26f1a94f36af 2024-11-17T21:24:25,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/c8ba42d3b18349a187ab26f1a94f36af, entries=200, sequenceid=438, filesize=14.4 K 2024-11-17T21:24:25,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/70814d2353ce407bacd1cd2171edc9af as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/70814d2353ce407bacd1cd2171edc9af 2024-11-17T21:24:25,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/70814d2353ce407bacd1cd2171edc9af, entries=150, sequenceid=438, filesize=12.0 K 2024-11-17T21:24:25,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/68f48769b71f4d53acc1b94d89321c3d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/68f48769b71f4d53acc1b94d89321c3d 2024-11-17T21:24:25,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/68f48769b71f4d53acc1b94d89321c3d, entries=150, sequenceid=438, filesize=12.0 K 2024-11-17T21:24:25,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 8417a211937373f3c01713928c5cb18b in 867ms, sequenceid=438, compaction requested=true 2024-11-17T21:24:25,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:25,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:25,831 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:25,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:25,831 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:25,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:25,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:25,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:25,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:25,834 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:25,834 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/B is initiating minor compaction (all files) 2024-11-17T21:24:25,834 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/B in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:25,834 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/bf6ce90f88b14f0aad0f3478a2e56c67, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0b19789033674b4193cabf84a51cfb3e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/70814d2353ce407bacd1cd2171edc9af] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=37.0 K 2024-11-17T21:24:25,834 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:25,834 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/A is initiating minor compaction (all files) 2024-11-17T21:24:25,834 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/A in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:25,834 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting bf6ce90f88b14f0aad0f3478a2e56c67, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731878661472 2024-11-17T21:24:25,835 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/d34dfc501dd44533a63bf9a5a6626367, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/08889fef91e849ce8e7171f202ed230f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/c8ba42d3b18349a187ab26f1a94f36af] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=39.4 K 2024-11-17T21:24:25,835 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b19789033674b4193cabf84a51cfb3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731878662662 2024-11-17T21:24:25,835 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting d34dfc501dd44533a63bf9a5a6626367, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731878661472 2024-11-17T21:24:25,835 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 70814d2353ce407bacd1cd2171edc9af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731878663822 2024-11-17T21:24:25,835 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08889fef91e849ce8e7171f202ed230f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731878662662 2024-11-17T21:24:25,836 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8ba42d3b18349a187ab26f1a94f36af, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731878663815 2024-11-17T21:24:25,844 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:25,844 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/5ff64400d89c45f4892579d313281b6d is 50, key is test_row_0/B:col10/1731878664962/Put/seqid=0 2024-11-17T21:24:25,846 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#A#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:25,846 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/86032a9ce9cf4867917dadf48eb95dc3 is 50, key is test_row_0/A:col10/1731878664962/Put/seqid=0 2024-11-17T21:24:25,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741931_1107 (size=13357) 2024-11-17T21:24:25,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741930_1106 (size=13357) 2024-11-17T21:24:25,874 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/86032a9ce9cf4867917dadf48eb95dc3 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/86032a9ce9cf4867917dadf48eb95dc3 2024-11-17T21:24:25,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T21:24:25,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:25,878 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T21:24:25,882 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/A of 8417a211937373f3c01713928c5cb18b into 86032a9ce9cf4867917dadf48eb95dc3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:25,883 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:25,883 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/A, priority=13, startTime=1731878665831; duration=0sec 2024-11-17T21:24:25,883 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:25,883 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:A 2024-11-17T21:24:25,883 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:25,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/45f4a633d4fb471bbb21935a5ee549e2 is 50, key is test_row_0/A:col10/1731878664986/Put/seqid=0 2024-11-17T21:24:25,885 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:25,885 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/C is initiating minor compaction (all files) 2024-11-17T21:24:25,885 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/C in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:25,885 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/84cb72b76e6242bc83bef2fc21247ce6, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/40c792ed46614c62acbc69fe3f453f99, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/68f48769b71f4d53acc1b94d89321c3d] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=37.0 K 2024-11-17T21:24:25,887 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84cb72b76e6242bc83bef2fc21247ce6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731878661472 2024-11-17T21:24:25,889 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40c792ed46614c62acbc69fe3f453f99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731878662662 2024-11-17T21:24:25,889 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68f48769b71f4d53acc1b94d89321c3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731878663822 2024-11-17T21:24:25,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741932_1108 (size=12301) 2024-11-17T21:24:25,898 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#C#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:25,898 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/d227e7ff3e474d36bb4eae6e22bfa945 is 50, key is test_row_0/C:col10/1731878664962/Put/seqid=0 2024-11-17T21:24:25,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741933_1109 (size=13357) 2024-11-17T21:24:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:26,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:26,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878726174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878726175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878726175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878726177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T21:24:26,272 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/5ff64400d89c45f4892579d313281b6d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/5ff64400d89c45f4892579d313281b6d 2024-11-17T21:24:26,278 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/B of 8417a211937373f3c01713928c5cb18b into 5ff64400d89c45f4892579d313281b6d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:26,278 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:26,278 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/B, priority=13, startTime=1731878665831; duration=0sec 2024-11-17T21:24:26,278 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:26,278 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:B 2024-11-17T21:24:26,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878726281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878726291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878726291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878726291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,295 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/45f4a633d4fb471bbb21935a5ee549e2 2024-11-17T21:24:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1e0e0001e2eb457eb7555583de0180bf is 50, key is test_row_0/B:col10/1731878664986/Put/seqid=0 2024-11-17T21:24:26,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741934_1110 (size=12301) 2024-11-17T21:24:26,313 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/d227e7ff3e474d36bb4eae6e22bfa945 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d227e7ff3e474d36bb4eae6e22bfa945 2024-11-17T21:24:26,320 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/C of 8417a211937373f3c01713928c5cb18b into d227e7ff3e474d36bb4eae6e22bfa945(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:26,320 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:26,320 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/C, priority=13, startTime=1731878665831; duration=0sec 2024-11-17T21:24:26,320 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:26,320 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:C 2024-11-17T21:24:26,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878726491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878726501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878726501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878726501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,713 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1e0e0001e2eb457eb7555583de0180bf 2024-11-17T21:24:26,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/5f75f1626149480d9075aecafe201151 is 50, key is test_row_0/C:col10/1731878664986/Put/seqid=0 2024-11-17T21:24:26,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741935_1111 (size=12301) 2024-11-17T21:24:26,756 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/5f75f1626149480d9075aecafe201151 2024-11-17T21:24:26,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/45f4a633d4fb471bbb21935a5ee549e2 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/45f4a633d4fb471bbb21935a5ee549e2 2024-11-17T21:24:26,770 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/45f4a633d4fb471bbb21935a5ee549e2, entries=150, sequenceid=454, filesize=12.0 K 2024-11-17T21:24:26,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1e0e0001e2eb457eb7555583de0180bf as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1e0e0001e2eb457eb7555583de0180bf 2024-11-17T21:24:26,777 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1e0e0001e2eb457eb7555583de0180bf, entries=150, sequenceid=454, filesize=12.0 K 2024-11-17T21:24:26,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/5f75f1626149480d9075aecafe201151 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/5f75f1626149480d9075aecafe201151 2024-11-17T21:24:26,783 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/5f75f1626149480d9075aecafe201151, entries=150, sequenceid=454, filesize=12.0 K 2024-11-17T21:24:26,784 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 8417a211937373f3c01713928c5cb18b in 906ms, sequenceid=454, compaction requested=false 2024-11-17T21:24:26,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:26,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:26,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-17T21:24:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-17T21:24:26,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-17T21:24:26,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6730 sec 2024-11-17T21:24:26,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.6780 sec 2024-11-17T21:24:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:26,803 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:26,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T21:24:26,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/8667bf846fde4baaba347a59333f43a0 is 50, key is test_row_0/A:col10/1731878666174/Put/seqid=0 2024-11-17T21:24:26,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741936_1112 (size=14741) 2024-11-17T21:24:26,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/8667bf846fde4baaba347a59333f43a0 2024-11-17T21:24:26,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/f5e8ed98ce374e48bfefa2b2a8b5a15f is 50, key is test_row_0/B:col10/1731878666174/Put/seqid=0 2024-11-17T21:24:26,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878726821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878726824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878726826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878726826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741937_1113 (size=12301) 2024-11-17T21:24:26,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878726931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878726931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878726931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:26,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878726941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:27,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878727141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:27,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:27,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878727141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:27,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878727142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:27,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:27,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878727151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:27,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/f5e8ed98ce374e48bfefa2b2a8b5a15f 2024-11-17T21:24:27,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/943558bfb42e4dec8f90f97e64ac92d2 is 50, key is test_row_0/C:col10/1731878666174/Put/seqid=0 2024-11-17T21:24:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T21:24:27,261 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:27,262 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-17T21:24:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T21:24:27,264 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:27,266 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:27,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:27,269 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43623 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=a313eea8709e,46445,1731878642828, table=TestAcidGuarantees, region=8417a211937373f3c01713928c5cb18b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-17T21:24:27,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741938_1114 (size=12301) 2024-11-17T21:24:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T21:24:27,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T21:24:27,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:27,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:27,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:27,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:27,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:27,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878727451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:27,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878727451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:27,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878727451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:27,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878727460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:27,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T21:24:27,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:27,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:27,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:27,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:27,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T21:24:27,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/943558bfb42e4dec8f90f97e64ac92d2 2024-11-17T21:24:27,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/8667bf846fde4baaba347a59333f43a0 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8667bf846fde4baaba347a59333f43a0 2024-11-17T21:24:27,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8667bf846fde4baaba347a59333f43a0, entries=200, sequenceid=478, filesize=14.4 K 2024-11-17T21:24:27,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/f5e8ed98ce374e48bfefa2b2a8b5a15f as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/f5e8ed98ce374e48bfefa2b2a8b5a15f 2024-11-17T21:24:27,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T21:24:27,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:27,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:27,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:27,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:27,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:27,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/f5e8ed98ce374e48bfefa2b2a8b5a15f, entries=150, sequenceid=478, filesize=12.0 K 2024-11-17T21:24:27,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/943558bfb42e4dec8f90f97e64ac92d2 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/943558bfb42e4dec8f90f97e64ac92d2 2024-11-17T21:24:27,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/943558bfb42e4dec8f90f97e64ac92d2, entries=150, sequenceid=478, filesize=12.0 K 2024-11-17T21:24:27,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 8417a211937373f3c01713928c5cb18b in 938ms, sequenceid=478, compaction requested=true 2024-11-17T21:24:27,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:27,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:27,741 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:27,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:27,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:27,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:27,741 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:27,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:27,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:27,742 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:27,742 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:27,742 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/A is initiating minor compaction (all files) 2024-11-17T21:24:27,742 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/B is initiating minor compaction (all files) 2024-11-17T21:24:27,742 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/B in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:27,742 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/A in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:27,743 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/5ff64400d89c45f4892579d313281b6d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1e0e0001e2eb457eb7555583de0180bf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/f5e8ed98ce374e48bfefa2b2a8b5a15f] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=37.1 K 2024-11-17T21:24:27,743 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/86032a9ce9cf4867917dadf48eb95dc3, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/45f4a633d4fb471bbb21935a5ee549e2, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8667bf846fde4baaba347a59333f43a0] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=39.5 K 2024-11-17T21:24:27,743 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ff64400d89c45f4892579d313281b6d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731878663822 2024-11-17T21:24:27,743 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86032a9ce9cf4867917dadf48eb95dc3, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731878663822 2024-11-17T21:24:27,743 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45f4a633d4fb471bbb21935a5ee549e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731878664978 2024-11-17T21:24:27,743 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e0e0001e2eb457eb7555583de0180bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731878664978 2024-11-17T21:24:27,744 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting f5e8ed98ce374e48bfefa2b2a8b5a15f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731878666174 2024-11-17T21:24:27,744 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8667bf846fde4baaba347a59333f43a0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731878666174 2024-11-17T21:24:27,758 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#A#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:27,758 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#B#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:27,758 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/b04f0a3d3cff4ee2978cd3b3a663d4c5 is 50, key is test_row_0/A:col10/1731878666174/Put/seqid=0 2024-11-17T21:24:27,758 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/a289e069b7c14763b7d92536a6753508 is 50, key is test_row_0/B:col10/1731878666174/Put/seqid=0 2024-11-17T21:24:27,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741939_1115 (size=13459) 2024-11-17T21:24:27,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741940_1116 (size=13459) 2024-11-17T21:24:27,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T21:24:27,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:27,886 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T21:24:27,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/65b6750a52e04d07b9a704418dc208ca is 50, key is test_row_0/A:col10/1731878666818/Put/seqid=0 2024-11-17T21:24:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T21:24:27,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741941_1117 (size=12301) 2024-11-17T21:24:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:27,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:27,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:27,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878727965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878727964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:27,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:27,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878727967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:27,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:27,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:27,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878727967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:27,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878727971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:27,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:27,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:27,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:27,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:28,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878728081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:28,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878728081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:28,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878728081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:28,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878728081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:28,206 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/b04f0a3d3cff4ee2978cd3b3a663d4c5 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b04f0a3d3cff4ee2978cd3b3a663d4c5 2024-11-17T21:24:28,210 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/a289e069b7c14763b7d92536a6753508 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/a289e069b7c14763b7d92536a6753508 2024-11-17T21:24:28,213 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/A of 8417a211937373f3c01713928c5cb18b into b04f0a3d3cff4ee2978cd3b3a663d4c5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:28,213 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:28,213 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/A, priority=13, startTime=1731878667741; duration=0sec 2024-11-17T21:24:28,213 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:28,213 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:A 2024-11-17T21:24:28,213 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:28,214 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:28,214 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/C is initiating minor compaction (all files) 2024-11-17T21:24:28,214 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/C in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:28,214 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d227e7ff3e474d36bb4eae6e22bfa945, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/5f75f1626149480d9075aecafe201151, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/943558bfb42e4dec8f90f97e64ac92d2] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=37.1 K 2024-11-17T21:24:28,215 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting d227e7ff3e474d36bb4eae6e22bfa945, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731878663822 2024-11-17T21:24:28,215 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f75f1626149480d9075aecafe201151, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731878664978 2024-11-17T21:24:28,215 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 943558bfb42e4dec8f90f97e64ac92d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731878666174 2024-11-17T21:24:28,217 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/B of 8417a211937373f3c01713928c5cb18b into a289e069b7c14763b7d92536a6753508(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:28,217 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:28,217 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/B, priority=13, startTime=1731878667741; duration=0sec 2024-11-17T21:24:28,217 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:28,217 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:B 2024-11-17T21:24:28,223 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#C#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:28,224 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/35d4af5a617d429fb90403949a1dfd10 is 50, key is test_row_0/C:col10/1731878666174/Put/seqid=0 2024-11-17T21:24:28,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741942_1118 (size=13459) 2024-11-17T21:24:28,239 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/35d4af5a617d429fb90403949a1dfd10 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/35d4af5a617d429fb90403949a1dfd10 2024-11-17T21:24:28,245 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/C of 8417a211937373f3c01713928c5cb18b into 35d4af5a617d429fb90403949a1dfd10(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:28,245 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:28,245 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/C, priority=13, startTime=1731878667741; duration=0sec 2024-11-17T21:24:28,245 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:28,245 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:C 2024-11-17T21:24:28,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878728291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:28,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878728291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:28,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:28,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878728292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:28,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:28,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878728292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:28,299 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/65b6750a52e04d07b9a704418dc208ca 2024-11-17T21:24:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/3ef49c83324a4429a22756a7fd5ced54 is 50, key is test_row_0/B:col10/1731878666818/Put/seqid=0 2024-11-17T21:24:28,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741943_1119 (size=12301) 2024-11-17T21:24:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T21:24:28,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878728602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:28,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:28,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878728602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:28,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878728602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:28,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878728602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:28,717 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/3ef49c83324a4429a22756a7fd5ced54 2024-11-17T21:24:28,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/ab544120200b4ad6b4d6c8a8500cca65 is 50, key is test_row_0/C:col10/1731878666818/Put/seqid=0 2024-11-17T21:24:28,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741944_1120 (size=12301) 2024-11-17T21:24:28,733 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/ab544120200b4ad6b4d6c8a8500cca65 2024-11-17T21:24:28,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/65b6750a52e04d07b9a704418dc208ca as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/65b6750a52e04d07b9a704418dc208ca 2024-11-17T21:24:28,748 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/65b6750a52e04d07b9a704418dc208ca, entries=150, sequenceid=494, filesize=12.0 K 2024-11-17T21:24:28,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/3ef49c83324a4429a22756a7fd5ced54 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/3ef49c83324a4429a22756a7fd5ced54 2024-11-17T21:24:28,755 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/3ef49c83324a4429a22756a7fd5ced54, entries=150, sequenceid=494, filesize=12.0 K 2024-11-17T21:24:28,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/ab544120200b4ad6b4d6c8a8500cca65 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/ab544120200b4ad6b4d6c8a8500cca65 2024-11-17T21:24:28,762 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/ab544120200b4ad6b4d6c8a8500cca65, entries=150, sequenceid=494, filesize=12.0 K 2024-11-17T21:24:28,763 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 8417a211937373f3c01713928c5cb18b in 877ms, sequenceid=494, compaction requested=false 2024-11-17T21:24:28,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:28,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:28,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-17T21:24:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-17T21:24:28,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-17T21:24:28,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5000 sec 2024-11-17T21:24:28,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.5080 sec 2024-11-17T21:24:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:28,985 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:28,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T21:24:28,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/6f33c081f518443cb66695b09acca0a1 is 50, key is test_row_0/A:col10/1731878668984/Put/seqid=0 2024-11-17T21:24:29,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741945_1121 (size=12301) 2024-11-17T21:24:29,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/6f33c081f518443cb66695b09acca0a1 2024-11-17T21:24:29,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/a3293d96a6e14745a0f3d60cc4496b9d is 50, key is test_row_0/B:col10/1731878668984/Put/seqid=0 2024-11-17T21:24:29,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878729043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:29,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:29,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741946_1122 (size=12301) 2024-11-17T21:24:29,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/a3293d96a6e14745a0f3d60cc4496b9d 2024-11-17T21:24:29,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/be4fa398051846f2ac7d56c08154b505 is 50, key is test_row_0/C:col10/1731878668984/Put/seqid=0 2024-11-17T21:24:29,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741947_1123 (size=12301) 2024-11-17T21:24:29,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/be4fa398051846f2ac7d56c08154b505 2024-11-17T21:24:29,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/6f33c081f518443cb66695b09acca0a1 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6f33c081f518443cb66695b09acca0a1 2024-11-17T21:24:29,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6f33c081f518443cb66695b09acca0a1, entries=150, sequenceid=518, filesize=12.0 K 2024-11-17T21:24:29,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/a3293d96a6e14745a0f3d60cc4496b9d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/a3293d96a6e14745a0f3d60cc4496b9d 2024-11-17T21:24:29,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/a3293d96a6e14745a0f3d60cc4496b9d, entries=150, sequenceid=518, filesize=12.0 K 2024-11-17T21:24:29,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/be4fa398051846f2ac7d56c08154b505 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/be4fa398051846f2ac7d56c08154b505 2024-11-17T21:24:29,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36102 deadline: 1731878729111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:29,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1731878729111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:29,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1731878729111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36080 deadline: 1731878729111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:29,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/be4fa398051846f2ac7d56c08154b505, entries=150, sequenceid=518, filesize=12.0 K 2024-11-17T21:24:29,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 8417a211937373f3c01713928c5cb18b in 137ms, sequenceid=518, compaction requested=true 2024-11-17T21:24:29,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:29,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:29,122 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:29,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:29,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:29,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:29,122 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:29,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:29,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:29,123 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:29,123 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:29,123 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/B is initiating minor compaction (all files) 2024-11-17T21:24:29,123 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/A is initiating minor compaction (all files) 2024-11-17T21:24:29,123 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/B in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:29,123 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/A in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:29,124 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/a289e069b7c14763b7d92536a6753508, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/3ef49c83324a4429a22756a7fd5ced54, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/a3293d96a6e14745a0f3d60cc4496b9d] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=37.2 K 2024-11-17T21:24:29,124 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b04f0a3d3cff4ee2978cd3b3a663d4c5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/65b6750a52e04d07b9a704418dc208ca, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6f33c081f518443cb66695b09acca0a1] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=37.2 K 2024-11-17T21:24:29,124 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting a289e069b7c14763b7d92536a6753508, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731878666174 2024-11-17T21:24:29,124 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting b04f0a3d3cff4ee2978cd3b3a663d4c5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731878666174 2024-11-17T21:24:29,125 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65b6750a52e04d07b9a704418dc208ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731878666818 2024-11-17T21:24:29,125 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ef49c83324a4429a22756a7fd5ced54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731878666818 2024-11-17T21:24:29,125 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting a3293d96a6e14745a0f3d60cc4496b9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731878667962 2024-11-17T21:24:29,125 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f33c081f518443cb66695b09acca0a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731878667962 2024-11-17T21:24:29,134 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#A#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:29,135 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/7518c3ce67ad41f093eb68c84d836788 is 50, key is test_row_0/A:col10/1731878668984/Put/seqid=0 2024-11-17T21:24:29,135 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#B#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:29,136 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/432d4ce7efab4bbcb7f50f1eea597e18 is 50, key is test_row_0/B:col10/1731878668984/Put/seqid=0 2024-11-17T21:24:29,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741949_1125 (size=13561) 2024-11-17T21:24:29,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741948_1124 (size=13561) 2024-11-17T21:24:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:29,153 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:29,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T21:24:29,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/8a0d7beed1e2474a96f30bf407d2d501 is 50, key is test_row_0/A:col10/1731878669152/Put/seqid=0 2024-11-17T21:24:29,170 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/7518c3ce67ad41f093eb68c84d836788 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/7518c3ce67ad41f093eb68c84d836788 2024-11-17T21:24:29,175 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/432d4ce7efab4bbcb7f50f1eea597e18 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/432d4ce7efab4bbcb7f50f1eea597e18 2024-11-17T21:24:29,186 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/A of 8417a211937373f3c01713928c5cb18b into 7518c3ce67ad41f093eb68c84d836788(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:29,186 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:29,186 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/A, priority=13, startTime=1731878669122; duration=0sec 2024-11-17T21:24:29,187 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:29,187 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:A 2024-11-17T21:24:29,187 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:29,189 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:29,189 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/C is initiating minor compaction (all files) 2024-11-17T21:24:29,189 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/C in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:29,189 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/35d4af5a617d429fb90403949a1dfd10, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/ab544120200b4ad6b4d6c8a8500cca65, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/be4fa398051846f2ac7d56c08154b505] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=37.2 K 2024-11-17T21:24:29,190 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35d4af5a617d429fb90403949a1dfd10, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731878666174 2024-11-17T21:24:29,190 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab544120200b4ad6b4d6c8a8500cca65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731878666818 2024-11-17T21:24:29,190 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting be4fa398051846f2ac7d56c08154b505, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731878667962 2024-11-17T21:24:29,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741950_1126 (size=14741) 2024-11-17T21:24:29,192 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/B of 8417a211937373f3c01713928c5cb18b into 432d4ce7efab4bbcb7f50f1eea597e18(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:29,192 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:29,192 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/B, priority=13, startTime=1731878669122; duration=0sec 2024-11-17T21:24:29,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/8a0d7beed1e2474a96f30bf407d2d501 2024-11-17T21:24:29,192 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:29,192 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:B 2024-11-17T21:24:29,207 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#C#compaction#115 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:29,208 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/e351fcbc54b74a768f7a5eba9f333a3d is 50, key is test_row_0/C:col10/1731878668984/Put/seqid=0 2024-11-17T21:24:29,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/9e9621751be94703bcb7e9c65ae4d9dd is 50, key is test_row_0/B:col10/1731878669152/Put/seqid=0 2024-11-17T21:24:29,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741952_1128 (size=12301) 2024-11-17T21:24:29,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741951_1127 (size=13561) 2024-11-17T21:24:29,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878729232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:29,237 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/e351fcbc54b74a768f7a5eba9f333a3d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/e351fcbc54b74a768f7a5eba9f333a3d 2024-11-17T21:24:29,243 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/C of 8417a211937373f3c01713928c5cb18b into e351fcbc54b74a768f7a5eba9f333a3d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:29,243 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:29,243 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/C, priority=13, startTime=1731878669122; duration=0sec 2024-11-17T21:24:29,243 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:29,243 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:C 2024-11-17T21:24:29,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878729341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:29,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T21:24:29,412 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:29,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-17T21:24:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T21:24:29,416 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:29,419 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:29,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T21:24:29,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878729550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:29,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T21:24:29,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:29,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:29,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:29,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:29,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:29,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/9e9621751be94703bcb7e9c65ae4d9dd 2024-11-17T21:24:29,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/fe829f19e7e440c4bb88a7b6d6bac695 is 50, key is test_row_0/C:col10/1731878669152/Put/seqid=0 2024-11-17T21:24:29,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741953_1129 (size=12301) 2024-11-17T21:24:29,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T21:24:29,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:29,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:29,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:29,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:29,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T21:24:29,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36116 deadline: 1731878729861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:24:29,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T21:24:29,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:29,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:29,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:29,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:29,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:29,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:29,961 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-17T21:24:29,961 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:29,961 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-17T21:24:29,962 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:29,962 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-17T21:24:29,962 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-17T21:24:29,962 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:29,962 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:29,966 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:29,966 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:29,966 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:29,966 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:29,967 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:29,967 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:29,967 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:29,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:29,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:29,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:29,967 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:29,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:30,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T21:24:30,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:30,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:30,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:30,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T21:24:30,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/fe829f19e7e440c4bb88a7b6d6bac695 2024-11-17T21:24:30,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/8a0d7beed1e2474a96f30bf407d2d501 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8a0d7beed1e2474a96f30bf407d2d501 2024-11-17T21:24:30,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8a0d7beed1e2474a96f30bf407d2d501, entries=200, sequenceid=534, filesize=14.4 K 2024-11-17T21:24:30,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/9e9621751be94703bcb7e9c65ae4d9dd as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/9e9621751be94703bcb7e9c65ae4d9dd 2024-11-17T21:24:30,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/9e9621751be94703bcb7e9c65ae4d9dd, entries=150, sequenceid=534, filesize=12.0 K 2024-11-17T21:24:30,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/fe829f19e7e440c4bb88a7b6d6bac695 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/fe829f19e7e440c4bb88a7b6d6bac695 2024-11-17T21:24:30,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/fe829f19e7e440c4bb88a7b6d6bac695, entries=150, sequenceid=534, filesize=12.0 K 2024-11-17T21:24:30,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 8417a211937373f3c01713928c5cb18b in 933ms, sequenceid=534, compaction requested=false 2024-11-17T21:24:30,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:30,122 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:30,122 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-17T21:24:30,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T21:24:30,122 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:30,122 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:30,122 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-17T21:24:30,122 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:30,122 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:30,122 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:30,122 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:30,122 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:30,122 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:30,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/ec46090e98e1457ea3c9d38229217338 is 50, key is test_row_0/A:col10/1731878670121/Put/seqid=0 2024-11-17T21:24:30,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741954_1130 (size=12301) 2024-11-17T21:24:30,133 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-17T21:24:30,133 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-17T21:24:30,133 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:30,133 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:30,133 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:30,133 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:30,133 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:30,133 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:30,133 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:30,133 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:30,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T21:24:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:30,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T21:24:30,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:30,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:30,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:30,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,374 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-17T21:24:30,374 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:30,374 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:30,374 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:30,375 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:30,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T21:24:30,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:30,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:30,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:30,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/ec46090e98e1457ea3c9d38229217338 2024-11-17T21:24:30,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/fb01d962ac2f42f9a2019a8a1a849d21 is 50, key is test_row_0/B:col10/1731878670121/Put/seqid=0 2024-11-17T21:24:30,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741955_1131 (size=12301) 2024-11-17T21:24:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T21:24:30,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T21:24:30,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:30,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:30,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:30,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T21:24:30,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:30,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:30,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:30,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/fb01d962ac2f42f9a2019a8a1a849d21 2024-11-17T21:24:30,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/396dbf9a70d04d53b31c2290123969e0 is 50, key is test_row_0/C:col10/1731878670121/Put/seqid=0 2024-11-17T21:24:30,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741956_1132 (size=12301) 2024-11-17T21:24:30,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T21:24:30,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:30,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:30,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:30,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:31,089 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T21:24:31,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T21:24:31,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:31,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:31,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:31,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:31,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:31,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T21:24:31,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:31,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. as already flushing 2024-11-17T21:24:31,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:31,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:31,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:31,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/396dbf9a70d04d53b31c2290123969e0 2024-11-17T21:24:31,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/ec46090e98e1457ea3c9d38229217338 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ec46090e98e1457ea3c9d38229217338 2024-11-17T21:24:31,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ec46090e98e1457ea3c9d38229217338, entries=150, sequenceid=559, filesize=12.0 K 2024-11-17T21:24:31,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/fb01d962ac2f42f9a2019a8a1a849d21 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/fb01d962ac2f42f9a2019a8a1a849d21 2024-11-17T21:24:31,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/fb01d962ac2f42f9a2019a8a1a849d21, entries=150, sequenceid=559, filesize=12.0 K 2024-11-17T21:24:31,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/396dbf9a70d04d53b31c2290123969e0 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/396dbf9a70d04d53b31c2290123969e0 2024-11-17T21:24:31,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/396dbf9a70d04d53b31c2290123969e0, entries=150, sequenceid=559, filesize=12.0 K 2024-11-17T21:24:31,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=20.13 KB/20610 for 8417a211937373f3c01713928c5cb18b in 1286ms, sequenceid=559, compaction requested=true 2024-11-17T21:24:31,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:31,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:31,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:31,409 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:31,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:31,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:31,409 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:31,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8417a211937373f3c01713928c5cb18b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:31,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:31,410 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:31,410 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40603 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:31,410 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/B is initiating minor compaction (all files) 2024-11-17T21:24:31,410 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/A is initiating minor compaction (all files) 2024-11-17T21:24:31,410 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/B in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:31,410 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/A in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:31,411 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/432d4ce7efab4bbcb7f50f1eea597e18, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/9e9621751be94703bcb7e9c65ae4d9dd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/fb01d962ac2f42f9a2019a8a1a849d21] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=37.3 K 2024-11-17T21:24:31,411 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/7518c3ce67ad41f093eb68c84d836788, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8a0d7beed1e2474a96f30bf407d2d501, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ec46090e98e1457ea3c9d38229217338] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=39.7 K 2024-11-17T21:24:31,411 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 432d4ce7efab4bbcb7f50f1eea597e18, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731878667962 2024-11-17T21:24:31,411 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7518c3ce67ad41f093eb68c84d836788, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731878667962 2024-11-17T21:24:31,412 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e9621751be94703bcb7e9c65ae4d9dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731878669024 2024-11-17T21:24:31,412 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a0d7beed1e2474a96f30bf407d2d501, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731878669024 2024-11-17T21:24:31,412 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting fb01d962ac2f42f9a2019a8a1a849d21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731878669208 2024-11-17T21:24:31,412 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec46090e98e1457ea3c9d38229217338, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731878669208 2024-11-17T21:24:31,421 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#B#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:31,421 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#A#compaction#121 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:31,422 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/8b45205c9eec46638f52b244d3a935c3 is 50, key is test_row_0/A:col10/1731878670121/Put/seqid=0 2024-11-17T21:24:31,422 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1329fd68d42545a99042a56e2b77396a is 50, key is test_row_0/B:col10/1731878670121/Put/seqid=0 2024-11-17T21:24:31,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741957_1133 (size=13663) 2024-11-17T21:24:31,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741958_1134 (size=13663) 2024-11-17T21:24:31,432 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/1329fd68d42545a99042a56e2b77396a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1329fd68d42545a99042a56e2b77396a 2024-11-17T21:24:31,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T21:24:31,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:31,432 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 8417a211937373f3c01713928c5cb18b 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-17T21:24:31,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/1d1d096eaf8047d2849a449ff7b5d228 is 50, key is test_row_1/A:col10/1731878670372/Put/seqid=0 2024-11-17T21:24:31,438 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/B of 8417a211937373f3c01713928c5cb18b into 1329fd68d42545a99042a56e2b77396a(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:31,438 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:31,438 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/B, priority=13, startTime=1731878671409; duration=0sec 2024-11-17T21:24:31,438 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:31,438 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:B 2024-11-17T21:24:31,438 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:31,439 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:31,439 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 8417a211937373f3c01713928c5cb18b/C is initiating minor compaction (all files) 2024-11-17T21:24:31,439 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8417a211937373f3c01713928c5cb18b/C in TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:31,439 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/e351fcbc54b74a768f7a5eba9f333a3d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/fe829f19e7e440c4bb88a7b6d6bac695, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/396dbf9a70d04d53b31c2290123969e0] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp, totalSize=37.3 K 2024-11-17T21:24:31,440 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting e351fcbc54b74a768f7a5eba9f333a3d, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731878667962 2024-11-17T21:24:31,440 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting fe829f19e7e440c4bb88a7b6d6bac695, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731878669024 2024-11-17T21:24:31,440 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 396dbf9a70d04d53b31c2290123969e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731878669208 2024-11-17T21:24:31,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741959_1135 (size=9857) 2024-11-17T21:24:31,447 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8417a211937373f3c01713928c5cb18b#C#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:31,448 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/c2a54add614a4eb18c100b616135ae9e is 50, key is test_row_0/C:col10/1731878670121/Put/seqid=0 2024-11-17T21:24:31,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741960_1136 (size=13663) 2024-11-17T21:24:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T21:24:31,841 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/8b45205c9eec46638f52b244d3a935c3 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8b45205c9eec46638f52b244d3a935c3 2024-11-17T21:24:31,841 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/1d1d096eaf8047d2849a449ff7b5d228 2024-11-17T21:24:31,849 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/A of 8417a211937373f3c01713928c5cb18b into 8b45205c9eec46638f52b244d3a935c3(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:31,849 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:31,849 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/A, priority=13, startTime=1731878671409; duration=0sec 2024-11-17T21:24:31,849 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:31,849 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:A 2024-11-17T21:24:31,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/7a1cc85cf96c472daa8b2eecbc2ba9cf is 50, key is test_row_1/B:col10/1731878670372/Put/seqid=0 2024-11-17T21:24:31,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741961_1137 (size=9857) 2024-11-17T21:24:31,858 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/c2a54add614a4eb18c100b616135ae9e as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c2a54add614a4eb18c100b616135ae9e 2024-11-17T21:24:31,865 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8417a211937373f3c01713928c5cb18b/C of 8417a211937373f3c01713928c5cb18b into c2a54add614a4eb18c100b616135ae9e(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:31,865 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:31,865 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b., storeName=8417a211937373f3c01713928c5cb18b/C, priority=13, startTime=1731878671409; duration=0sec 2024-11-17T21:24:31,865 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:31,865 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8417a211937373f3c01713928c5cb18b:C 2024-11-17T21:24:32,259 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/7a1cc85cf96c472daa8b2eecbc2ba9cf 2024-11-17T21:24:32,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/39ec3dbf554a421dbaeb8b6ae1dc83a7 is 50, key is test_row_1/C:col10/1731878670372/Put/seqid=0 2024-11-17T21:24:32,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741962_1138 (size=9857) 2024-11-17T21:24:32,680 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/39ec3dbf554a421dbaeb8b6ae1dc83a7 2024-11-17T21:24:32,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/A/1d1d096eaf8047d2849a449ff7b5d228 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/1d1d096eaf8047d2849a449ff7b5d228 2024-11-17T21:24:32,754 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/1d1d096eaf8047d2849a449ff7b5d228, entries=100, sequenceid=565, filesize=9.6 K 2024-11-17T21:24:32,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/B/7a1cc85cf96c472daa8b2eecbc2ba9cf as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/7a1cc85cf96c472daa8b2eecbc2ba9cf 2024-11-17T21:24:32,762 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/7a1cc85cf96c472daa8b2eecbc2ba9cf, entries=100, sequenceid=565, filesize=9.6 K 2024-11-17T21:24:32,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/.tmp/C/39ec3dbf554a421dbaeb8b6ae1dc83a7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/39ec3dbf554a421dbaeb8b6ae1dc83a7 2024-11-17T21:24:32,768 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/39ec3dbf554a421dbaeb8b6ae1dc83a7, entries=100, sequenceid=565, filesize=9.6 K 2024-11-17T21:24:32,769 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 8417a211937373f3c01713928c5cb18b in 1337ms, sequenceid=565, compaction requested=false 2024-11-17T21:24:32,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 8417a211937373f3c01713928c5cb18b: 2024-11-17T21:24:32,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:32,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-17T21:24:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-17T21:24:32,772 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-17T21:24:32,772 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3510 sec 2024-11-17T21:24:32,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 3.3610 sec 2024-11-17T21:24:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T21:24:33,571 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:33,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T21:24:33,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-17T21:24:33,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-17T21:24:33,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-17T21:24:33,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-17T21:24:33,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-17T21:24:33,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T21:24:33,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3849 2024-11-17T21:24:33,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3776 2024-11-17T21:24:33,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T21:24:33,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1720 2024-11-17T21:24:33,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5160 rows 2024-11-17T21:24:33,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1716 2024-11-17T21:24:33,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5148 rows 2024-11-17T21:24:33,571 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T21:24:33,571 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T21:24:33,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:33,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:33,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:33,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T21:24:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T21:24:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T21:24:33,581 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878673581"}]},"ts":"1731878673581"} 2024-11-17T21:24:33,583 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T21:24:33,583 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T21:24:33,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T21:24:33,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8417a211937373f3c01713928c5cb18b, UNASSIGN}] 2024-11-17T21:24:33,590 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8417a211937373f3c01713928c5cb18b, UNASSIGN 2024-11-17T21:24:33,592 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=8417a211937373f3c01713928c5cb18b, regionState=CLOSING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:24:33,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8417a211937373f3c01713928c5cb18b, UNASSIGN because future has completed 2024-11-17T21:24:33,594 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T21:24:33,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:24:33,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T21:24:33,754 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:33,754 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T21:24:33,756 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 8417a211937373f3c01713928c5cb18b, disabling compactions & flushes 2024-11-17T21:24:33,756 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:33,756 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:33,756 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. after waiting 0 ms 2024-11-17T21:24:33,756 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:33,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/cf96fe7346bd4dba80dd031dbc7a2a94, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/fa8d1d43d29c4eceb16372cb60628e78, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/dc56e80ac0ec4924b42ca73d2d8924f6, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ca2a994bdbbb462a8208b5e4889e9a3a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8816da2ca1af411ab4c274e9198ea6d1, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ed69eecc2d14452b85c4e9ae96d8a939, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/3a51f5b5adc941529b62d9fc9278605b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/41e207a747de49f1b724d8de9ad675f2, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/a7bc0b566bb54ae3bb73199fc7dd4732, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6c0493f65e074be59cca38be28d7ff62, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/f79d1b75035c456ca42fd0cf97644894, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/1f0358ce43394ccc95f89d9e10aeda1e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/a643a105c2d84435b93d6eba61bced63, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/f147b9a21fd34b7099bff28f49f08669, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b2ae0225c58b4f84bcf5fe6b6ac95595, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b432554b3c8a4cb1a39b0599b2030812, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/dabed9892c3343be97c694cf9df6c02f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/90878f5b18e54f0997343f20ae7b1e6d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/355fdda0fef8456b8ac5d90b3d46fd8b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/9b557f31e586441bb6202f1db5977c51, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/696fee97238048c3b880d9a1b86b1dca, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/37e861997d2e45d3854e92f2c7c60578, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/40056f446fc94af3adecb3959b9cc951, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6885f522193c4ed0b228fed312ad4082, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/08a67300a9954155adfd8b03225f2e29, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/79f28ad6193e4a39ae8109657b75be92, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b16ac52a46d9489fbed907053a02deab, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/d34dfc501dd44533a63bf9a5a6626367, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/3f3b1624f9054b3e8c168d117746ba72, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/08889fef91e849ce8e7171f202ed230f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/c8ba42d3b18349a187ab26f1a94f36af, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/86032a9ce9cf4867917dadf48eb95dc3, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/45f4a633d4fb471bbb21935a5ee549e2, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8667bf846fde4baaba347a59333f43a0, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b04f0a3d3cff4ee2978cd3b3a663d4c5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/65b6750a52e04d07b9a704418dc208ca, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/7518c3ce67ad41f093eb68c84d836788, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6f33c081f518443cb66695b09acca0a1, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8a0d7beed1e2474a96f30bf407d2d501, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ec46090e98e1457ea3c9d38229217338] to archive 2024-11-17T21:24:33,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:24:33,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/cf96fe7346bd4dba80dd031dbc7a2a94 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/cf96fe7346bd4dba80dd031dbc7a2a94 2024-11-17T21:24:33,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/fa8d1d43d29c4eceb16372cb60628e78 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/fa8d1d43d29c4eceb16372cb60628e78 2024-11-17T21:24:33,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/dc56e80ac0ec4924b42ca73d2d8924f6 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/dc56e80ac0ec4924b42ca73d2d8924f6 2024-11-17T21:24:33,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ca2a994bdbbb462a8208b5e4889e9a3a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ca2a994bdbbb462a8208b5e4889e9a3a 2024-11-17T21:24:33,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8816da2ca1af411ab4c274e9198ea6d1 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8816da2ca1af411ab4c274e9198ea6d1 2024-11-17T21:24:33,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ed69eecc2d14452b85c4e9ae96d8a939 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ed69eecc2d14452b85c4e9ae96d8a939 2024-11-17T21:24:33,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/3a51f5b5adc941529b62d9fc9278605b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/3a51f5b5adc941529b62d9fc9278605b 2024-11-17T21:24:33,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/41e207a747de49f1b724d8de9ad675f2 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/41e207a747de49f1b724d8de9ad675f2 2024-11-17T21:24:33,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/a7bc0b566bb54ae3bb73199fc7dd4732 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/a7bc0b566bb54ae3bb73199fc7dd4732 2024-11-17T21:24:33,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6c0493f65e074be59cca38be28d7ff62 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6c0493f65e074be59cca38be28d7ff62 2024-11-17T21:24:33,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/f79d1b75035c456ca42fd0cf97644894 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/f79d1b75035c456ca42fd0cf97644894 2024-11-17T21:24:33,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/1f0358ce43394ccc95f89d9e10aeda1e to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/1f0358ce43394ccc95f89d9e10aeda1e 2024-11-17T21:24:33,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/a643a105c2d84435b93d6eba61bced63 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/a643a105c2d84435b93d6eba61bced63 2024-11-17T21:24:33,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/f147b9a21fd34b7099bff28f49f08669 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/f147b9a21fd34b7099bff28f49f08669 2024-11-17T21:24:33,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b2ae0225c58b4f84bcf5fe6b6ac95595 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b2ae0225c58b4f84bcf5fe6b6ac95595 2024-11-17T21:24:33,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b432554b3c8a4cb1a39b0599b2030812 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b432554b3c8a4cb1a39b0599b2030812 2024-11-17T21:24:33,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/dabed9892c3343be97c694cf9df6c02f to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/dabed9892c3343be97c694cf9df6c02f 2024-11-17T21:24:33,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/90878f5b18e54f0997343f20ae7b1e6d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/90878f5b18e54f0997343f20ae7b1e6d 2024-11-17T21:24:33,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/355fdda0fef8456b8ac5d90b3d46fd8b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/355fdda0fef8456b8ac5d90b3d46fd8b 2024-11-17T21:24:33,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/9b557f31e586441bb6202f1db5977c51 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/9b557f31e586441bb6202f1db5977c51 2024-11-17T21:24:33,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/696fee97238048c3b880d9a1b86b1dca to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/696fee97238048c3b880d9a1b86b1dca 2024-11-17T21:24:33,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/37e861997d2e45d3854e92f2c7c60578 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/37e861997d2e45d3854e92f2c7c60578 2024-11-17T21:24:33,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/40056f446fc94af3adecb3959b9cc951 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/40056f446fc94af3adecb3959b9cc951 2024-11-17T21:24:33,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6885f522193c4ed0b228fed312ad4082 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6885f522193c4ed0b228fed312ad4082 2024-11-17T21:24:33,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/08a67300a9954155adfd8b03225f2e29 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/08a67300a9954155adfd8b03225f2e29 2024-11-17T21:24:33,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/79f28ad6193e4a39ae8109657b75be92 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/79f28ad6193e4a39ae8109657b75be92 2024-11-17T21:24:33,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b16ac52a46d9489fbed907053a02deab to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b16ac52a46d9489fbed907053a02deab 2024-11-17T21:24:33,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/d34dfc501dd44533a63bf9a5a6626367 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/d34dfc501dd44533a63bf9a5a6626367 2024-11-17T21:24:33,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/3f3b1624f9054b3e8c168d117746ba72 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/3f3b1624f9054b3e8c168d117746ba72 2024-11-17T21:24:33,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/08889fef91e849ce8e7171f202ed230f to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/08889fef91e849ce8e7171f202ed230f 2024-11-17T21:24:33,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/c8ba42d3b18349a187ab26f1a94f36af to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/c8ba42d3b18349a187ab26f1a94f36af 2024-11-17T21:24:33,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/86032a9ce9cf4867917dadf48eb95dc3 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/86032a9ce9cf4867917dadf48eb95dc3 2024-11-17T21:24:33,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/45f4a633d4fb471bbb21935a5ee549e2 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/45f4a633d4fb471bbb21935a5ee549e2 2024-11-17T21:24:33,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8667bf846fde4baaba347a59333f43a0 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8667bf846fde4baaba347a59333f43a0 2024-11-17T21:24:33,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b04f0a3d3cff4ee2978cd3b3a663d4c5 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/b04f0a3d3cff4ee2978cd3b3a663d4c5 2024-11-17T21:24:33,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/65b6750a52e04d07b9a704418dc208ca to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/65b6750a52e04d07b9a704418dc208ca 2024-11-17T21:24:33,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/7518c3ce67ad41f093eb68c84d836788 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/7518c3ce67ad41f093eb68c84d836788 2024-11-17T21:24:33,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6f33c081f518443cb66695b09acca0a1 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/6f33c081f518443cb66695b09acca0a1 2024-11-17T21:24:33,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8a0d7beed1e2474a96f30bf407d2d501 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8a0d7beed1e2474a96f30bf407d2d501 2024-11-17T21:24:33,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ec46090e98e1457ea3c9d38229217338 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/ec46090e98e1457ea3c9d38229217338 2024-11-17T21:24:33,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1259e240cff849758d2e66eb1abc5140, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/55bdfe53ccf84a269bd32023b1e00ec7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/8985412ee2a944d5a41ad36a3dbc5869, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/de2f4cd2d0c14e71a8d217506c9093dd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/be5d03076c324d3a8499fb9fad634c32, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/b6f7f6a22c254a03b8fab0392cd1e8fa, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/23b39a5291f34f80b7dd560ba0357626, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1f4f2a40aa4143378218f9e2015b2f1b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/5b8b43ab925c4279b6309cc6f878955a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/d65e680d6e7843c9a851e9748ce670f1, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/7e4ab2b2b39243a88e5c2982ebf80ea5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/598e6d2248dd4517bfd338910d27d658, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1236fa9e6b384b21906fb665f4946185, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0aa8643aea4b46a19c60b4c245fc2cf3, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/60a59c50175e4c44a7db94bc527aadd5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1a7b509ba72b4be8944da82de9b6348d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/64e499bc9b3040ca989d3181e6db872a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/c8c16adaa19145fb97569c53ceca90f3, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/ba825cfb48904d40bd8891e6916fc0d1, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/e04112f3dde24c63ad8ea14a91de5208, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/bc3ff480c1a64e67a0a7d8af66b6fc4c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/9eeb7669b7594513babd4f477974b783, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/38117ac7868643e58cb83b05efbc5d89, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/36dd30778ca74d13a8dad620c2502ef2, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/7be046f6ea774456a59c2bedfd2ac237, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/f44ea9d05b6d4ee09129649aa3502246, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/4713356991cc46d69222d53a7546743d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/bf6ce90f88b14f0aad0f3478a2e56c67, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0015c2646e8c4c79b403d994b305c4dd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0b19789033674b4193cabf84a51cfb3e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/5ff64400d89c45f4892579d313281b6d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/70814d2353ce407bacd1cd2171edc9af, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1e0e0001e2eb457eb7555583de0180bf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/a289e069b7c14763b7d92536a6753508, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/f5e8ed98ce374e48bfefa2b2a8b5a15f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/3ef49c83324a4429a22756a7fd5ced54, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/432d4ce7efab4bbcb7f50f1eea597e18, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/a3293d96a6e14745a0f3d60cc4496b9d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/9e9621751be94703bcb7e9c65ae4d9dd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/fb01d962ac2f42f9a2019a8a1a849d21] to archive 2024-11-17T21:24:33,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:24:33,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1259e240cff849758d2e66eb1abc5140 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1259e240cff849758d2e66eb1abc5140 2024-11-17T21:24:33,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/55bdfe53ccf84a269bd32023b1e00ec7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/55bdfe53ccf84a269bd32023b1e00ec7 2024-11-17T21:24:33,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/8985412ee2a944d5a41ad36a3dbc5869 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/8985412ee2a944d5a41ad36a3dbc5869 2024-11-17T21:24:33,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/de2f4cd2d0c14e71a8d217506c9093dd to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/de2f4cd2d0c14e71a8d217506c9093dd 2024-11-17T21:24:33,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/be5d03076c324d3a8499fb9fad634c32 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/be5d03076c324d3a8499fb9fad634c32 2024-11-17T21:24:33,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/b6f7f6a22c254a03b8fab0392cd1e8fa to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/b6f7f6a22c254a03b8fab0392cd1e8fa 2024-11-17T21:24:33,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/23b39a5291f34f80b7dd560ba0357626 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/23b39a5291f34f80b7dd560ba0357626 2024-11-17T21:24:33,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1f4f2a40aa4143378218f9e2015b2f1b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1f4f2a40aa4143378218f9e2015b2f1b 2024-11-17T21:24:33,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/5b8b43ab925c4279b6309cc6f878955a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/5b8b43ab925c4279b6309cc6f878955a 2024-11-17T21:24:33,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/d65e680d6e7843c9a851e9748ce670f1 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/d65e680d6e7843c9a851e9748ce670f1 2024-11-17T21:24:33,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/7e4ab2b2b39243a88e5c2982ebf80ea5 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/7e4ab2b2b39243a88e5c2982ebf80ea5 2024-11-17T21:24:33,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/598e6d2248dd4517bfd338910d27d658 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/598e6d2248dd4517bfd338910d27d658 2024-11-17T21:24:33,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1236fa9e6b384b21906fb665f4946185 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1236fa9e6b384b21906fb665f4946185 2024-11-17T21:24:33,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0aa8643aea4b46a19c60b4c245fc2cf3 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0aa8643aea4b46a19c60b4c245fc2cf3 2024-11-17T21:24:33,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/60a59c50175e4c44a7db94bc527aadd5 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/60a59c50175e4c44a7db94bc527aadd5 2024-11-17T21:24:33,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1a7b509ba72b4be8944da82de9b6348d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1a7b509ba72b4be8944da82de9b6348d 2024-11-17T21:24:33,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/64e499bc9b3040ca989d3181e6db872a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/64e499bc9b3040ca989d3181e6db872a 2024-11-17T21:24:33,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/c8c16adaa19145fb97569c53ceca90f3 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/c8c16adaa19145fb97569c53ceca90f3 2024-11-17T21:24:33,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/ba825cfb48904d40bd8891e6916fc0d1 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/ba825cfb48904d40bd8891e6916fc0d1 2024-11-17T21:24:33,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/e04112f3dde24c63ad8ea14a91de5208 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/e04112f3dde24c63ad8ea14a91de5208 2024-11-17T21:24:33,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/bc3ff480c1a64e67a0a7d8af66b6fc4c to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/bc3ff480c1a64e67a0a7d8af66b6fc4c 2024-11-17T21:24:33,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/9eeb7669b7594513babd4f477974b783 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/9eeb7669b7594513babd4f477974b783 2024-11-17T21:24:33,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/38117ac7868643e58cb83b05efbc5d89 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/38117ac7868643e58cb83b05efbc5d89 2024-11-17T21:24:33,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/36dd30778ca74d13a8dad620c2502ef2 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/36dd30778ca74d13a8dad620c2502ef2 2024-11-17T21:24:33,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/7be046f6ea774456a59c2bedfd2ac237 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/7be046f6ea774456a59c2bedfd2ac237 2024-11-17T21:24:33,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/f44ea9d05b6d4ee09129649aa3502246 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/f44ea9d05b6d4ee09129649aa3502246 2024-11-17T21:24:33,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/4713356991cc46d69222d53a7546743d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/4713356991cc46d69222d53a7546743d 2024-11-17T21:24:33,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/bf6ce90f88b14f0aad0f3478a2e56c67 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/bf6ce90f88b14f0aad0f3478a2e56c67 2024-11-17T21:24:33,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0015c2646e8c4c79b403d994b305c4dd to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0015c2646e8c4c79b403d994b305c4dd 2024-11-17T21:24:33,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0b19789033674b4193cabf84a51cfb3e to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/0b19789033674b4193cabf84a51cfb3e 2024-11-17T21:24:33,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/5ff64400d89c45f4892579d313281b6d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/5ff64400d89c45f4892579d313281b6d 2024-11-17T21:24:33,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/70814d2353ce407bacd1cd2171edc9af to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/70814d2353ce407bacd1cd2171edc9af 2024-11-17T21:24:33,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1e0e0001e2eb457eb7555583de0180bf to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1e0e0001e2eb457eb7555583de0180bf 2024-11-17T21:24:33,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/a289e069b7c14763b7d92536a6753508 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/a289e069b7c14763b7d92536a6753508 2024-11-17T21:24:33,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/f5e8ed98ce374e48bfefa2b2a8b5a15f to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/f5e8ed98ce374e48bfefa2b2a8b5a15f 2024-11-17T21:24:33,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/3ef49c83324a4429a22756a7fd5ced54 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/3ef49c83324a4429a22756a7fd5ced54 2024-11-17T21:24:33,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/432d4ce7efab4bbcb7f50f1eea597e18 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/432d4ce7efab4bbcb7f50f1eea597e18 2024-11-17T21:24:33,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/a3293d96a6e14745a0f3d60cc4496b9d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/a3293d96a6e14745a0f3d60cc4496b9d 2024-11-17T21:24:33,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/9e9621751be94703bcb7e9c65ae4d9dd to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/9e9621751be94703bcb7e9c65ae4d9dd 2024-11-17T21:24:33,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/fb01d962ac2f42f9a2019a8a1a849d21 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/fb01d962ac2f42f9a2019a8a1a849d21 2024-11-17T21:24:33,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/3bcf53c78a214ffb8a903d380027872c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/0f36ea8f282b4994acb696e07799ac83, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/3d63f573c2874a538d1b123b456d5406, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1704fd7fde514c14bc58582e482a0f9f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/4a4161a33d024c568a5735a1a4f0b934, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/83b89d7ce86b4141a766b9eda57d5535, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a037c0be97274a86bd9478e1c2325d06, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/cd00ff8d07d840aaaff71129ba978baf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/ce2890b8a7be425fb2ec3dcabd0a0f15, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1f7a86417d2c4a199b1e1fbf702de61a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/bb0ae0dadbc6471f9020df12abad8064, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d65710282e8b41b98c87f5f6c7006693, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c84a005c8e424d6b9d0d83c95063e4a5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1f08018227ce48ffbc6f0f191b29f9ad, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/8790fc40da3c4ee0a8d8051602d415f8, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/08ae7c27a9db45a5a50562dc8bcfa83d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/4dbcba52061948b088390c408f3240b1, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/3c365229a8e64b6eb841ea2d9faff8ef, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a9b38167405f495aa140432f622ef377, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1fb7f9175fe34de6a535b02ab235b666, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a56a3393798c4214ba45954b22ae7197, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c23be52488814173853db70a92f22d16, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d967f90cfec344b4b894cd6bda2e4d10, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d44255eea0c346f9997810985e3e9e00, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c6035e45dbbb44369b988633add2a5b9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/6c76b115aa454604a1c9b92af0aa915c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/44b3ed94c0b54c878a20e4d03a5b2f5b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/84cb72b76e6242bc83bef2fc21247ce6, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/5b204c639d3043dfa53df4f00e81c27c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/40c792ed46614c62acbc69fe3f453f99, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d227e7ff3e474d36bb4eae6e22bfa945, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/68f48769b71f4d53acc1b94d89321c3d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/5f75f1626149480d9075aecafe201151, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/35d4af5a617d429fb90403949a1dfd10, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/943558bfb42e4dec8f90f97e64ac92d2, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/ab544120200b4ad6b4d6c8a8500cca65, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/e351fcbc54b74a768f7a5eba9f333a3d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/be4fa398051846f2ac7d56c08154b505, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/fe829f19e7e440c4bb88a7b6d6bac695, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/396dbf9a70d04d53b31c2290123969e0] to archive 2024-11-17T21:24:33,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:24:33,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/3bcf53c78a214ffb8a903d380027872c to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/3bcf53c78a214ffb8a903d380027872c 2024-11-17T21:24:33,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/0f36ea8f282b4994acb696e07799ac83 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/0f36ea8f282b4994acb696e07799ac83 2024-11-17T21:24:33,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/3d63f573c2874a538d1b123b456d5406 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/3d63f573c2874a538d1b123b456d5406 2024-11-17T21:24:33,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1704fd7fde514c14bc58582e482a0f9f to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1704fd7fde514c14bc58582e482a0f9f 2024-11-17T21:24:33,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/4a4161a33d024c568a5735a1a4f0b934 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/4a4161a33d024c568a5735a1a4f0b934 2024-11-17T21:24:33,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/83b89d7ce86b4141a766b9eda57d5535 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/83b89d7ce86b4141a766b9eda57d5535 2024-11-17T21:24:33,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a037c0be97274a86bd9478e1c2325d06 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a037c0be97274a86bd9478e1c2325d06 2024-11-17T21:24:33,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/cd00ff8d07d840aaaff71129ba978baf to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/cd00ff8d07d840aaaff71129ba978baf 2024-11-17T21:24:33,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/ce2890b8a7be425fb2ec3dcabd0a0f15 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/ce2890b8a7be425fb2ec3dcabd0a0f15 2024-11-17T21:24:33,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1f7a86417d2c4a199b1e1fbf702de61a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1f7a86417d2c4a199b1e1fbf702de61a 2024-11-17T21:24:33,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/bb0ae0dadbc6471f9020df12abad8064 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/bb0ae0dadbc6471f9020df12abad8064 2024-11-17T21:24:33,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d65710282e8b41b98c87f5f6c7006693 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d65710282e8b41b98c87f5f6c7006693 2024-11-17T21:24:33,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c84a005c8e424d6b9d0d83c95063e4a5 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c84a005c8e424d6b9d0d83c95063e4a5 2024-11-17T21:24:33,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1f08018227ce48ffbc6f0f191b29f9ad to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1f08018227ce48ffbc6f0f191b29f9ad 2024-11-17T21:24:33,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/8790fc40da3c4ee0a8d8051602d415f8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/8790fc40da3c4ee0a8d8051602d415f8 2024-11-17T21:24:33,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/08ae7c27a9db45a5a50562dc8bcfa83d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/08ae7c27a9db45a5a50562dc8bcfa83d 2024-11-17T21:24:33,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/4dbcba52061948b088390c408f3240b1 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/4dbcba52061948b088390c408f3240b1 2024-11-17T21:24:33,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/3c365229a8e64b6eb841ea2d9faff8ef to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/3c365229a8e64b6eb841ea2d9faff8ef 2024-11-17T21:24:33,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a9b38167405f495aa140432f622ef377 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a9b38167405f495aa140432f622ef377 2024-11-17T21:24:33,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1fb7f9175fe34de6a535b02ab235b666 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/1fb7f9175fe34de6a535b02ab235b666 2024-11-17T21:24:33,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a56a3393798c4214ba45954b22ae7197 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/a56a3393798c4214ba45954b22ae7197 2024-11-17T21:24:33,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c23be52488814173853db70a92f22d16 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c23be52488814173853db70a92f22d16 2024-11-17T21:24:33,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d967f90cfec344b4b894cd6bda2e4d10 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d967f90cfec344b4b894cd6bda2e4d10 2024-11-17T21:24:33,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d44255eea0c346f9997810985e3e9e00 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d44255eea0c346f9997810985e3e9e00 2024-11-17T21:24:33,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c6035e45dbbb44369b988633add2a5b9 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c6035e45dbbb44369b988633add2a5b9 2024-11-17T21:24:33,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/6c76b115aa454604a1c9b92af0aa915c to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/6c76b115aa454604a1c9b92af0aa915c 2024-11-17T21:24:33,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/44b3ed94c0b54c878a20e4d03a5b2f5b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/44b3ed94c0b54c878a20e4d03a5b2f5b 2024-11-17T21:24:33,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/84cb72b76e6242bc83bef2fc21247ce6 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/84cb72b76e6242bc83bef2fc21247ce6 2024-11-17T21:24:33,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/5b204c639d3043dfa53df4f00e81c27c to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/5b204c639d3043dfa53df4f00e81c27c 2024-11-17T21:24:33,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/40c792ed46614c62acbc69fe3f453f99 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/40c792ed46614c62acbc69fe3f453f99 2024-11-17T21:24:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T21:24:33,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d227e7ff3e474d36bb4eae6e22bfa945 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/d227e7ff3e474d36bb4eae6e22bfa945 2024-11-17T21:24:33,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/68f48769b71f4d53acc1b94d89321c3d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/68f48769b71f4d53acc1b94d89321c3d 2024-11-17T21:24:33,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/5f75f1626149480d9075aecafe201151 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/5f75f1626149480d9075aecafe201151 2024-11-17T21:24:33,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/35d4af5a617d429fb90403949a1dfd10 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/35d4af5a617d429fb90403949a1dfd10 2024-11-17T21:24:33,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/943558bfb42e4dec8f90f97e64ac92d2 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/943558bfb42e4dec8f90f97e64ac92d2 2024-11-17T21:24:33,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/ab544120200b4ad6b4d6c8a8500cca65 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/ab544120200b4ad6b4d6c8a8500cca65 2024-11-17T21:24:33,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/e351fcbc54b74a768f7a5eba9f333a3d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/e351fcbc54b74a768f7a5eba9f333a3d 2024-11-17T21:24:33,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/be4fa398051846f2ac7d56c08154b505 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/be4fa398051846f2ac7d56c08154b505 2024-11-17T21:24:33,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/fe829f19e7e440c4bb88a7b6d6bac695 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/fe829f19e7e440c4bb88a7b6d6bac695 2024-11-17T21:24:33,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/396dbf9a70d04d53b31c2290123969e0 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/396dbf9a70d04d53b31c2290123969e0 2024-11-17T21:24:33,919 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/recovered.edits/571.seqid, newMaxSeqId=571, maxSeqId=1 2024-11-17T21:24:33,921 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b. 2024-11-17T21:24:33,922 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 8417a211937373f3c01713928c5cb18b: Waiting for close lock at 1731878673756Running coprocessor pre-close hooks at 1731878673756Disabling compacts and flushes for region at 1731878673756Disabling writes for close at 1731878673756Writing region close event to WAL at 1731878673915 (+159 ms)Running coprocessor post-close hooks at 1731878673920 (+5 ms)Closed at 1731878673921 (+1 ms) 2024-11-17T21:24:33,924 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:33,924 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=8417a211937373f3c01713928c5cb18b, regionState=CLOSED 2024-11-17T21:24:33,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:24:33,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-17T21:24:33,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 8417a211937373f3c01713928c5cb18b, server=a313eea8709e,46445,1731878642828 in 332 msec 2024-11-17T21:24:33,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-17T21:24:33,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8417a211937373f3c01713928c5cb18b, UNASSIGN in 339 msec 2024-11-17T21:24:33,931 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-17T21:24:33,931 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 345 msec 2024-11-17T21:24:33,933 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878673933"}]},"ts":"1731878673933"} 2024-11-17T21:24:33,934 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T21:24:33,934 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T21:24:33,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 359 msec 2024-11-17T21:24:34,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T21:24:34,212 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:34,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T21:24:34,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:24:34,226 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:24:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-17T21:24:34,227 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:24:34,231 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:34,235 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/recovered.edits] 2024-11-17T21:24:34,239 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/1d1d096eaf8047d2849a449ff7b5d228 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/1d1d096eaf8047d2849a449ff7b5d228 2024-11-17T21:24:34,240 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8b45205c9eec46638f52b244d3a935c3 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/A/8b45205c9eec46638f52b244d3a935c3 2024-11-17T21:24:34,243 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1329fd68d42545a99042a56e2b77396a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/1329fd68d42545a99042a56e2b77396a 2024-11-17T21:24:34,244 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/7a1cc85cf96c472daa8b2eecbc2ba9cf to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/B/7a1cc85cf96c472daa8b2eecbc2ba9cf 2024-11-17T21:24:34,246 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/39ec3dbf554a421dbaeb8b6ae1dc83a7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/39ec3dbf554a421dbaeb8b6ae1dc83a7 2024-11-17T21:24:34,247 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c2a54add614a4eb18c100b616135ae9e to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/C/c2a54add614a4eb18c100b616135ae9e 2024-11-17T21:24:34,249 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/recovered.edits/571.seqid to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b/recovered.edits/571.seqid 2024-11-17T21:24:34,250 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/8417a211937373f3c01713928c5cb18b 2024-11-17T21:24:34,250 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T21:24:34,254 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:24:34,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-17T21:24:34,262 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T21:24:34,295 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T21:24:34,297 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:24:34,297 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T21:24:34,298 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731878674297"}]},"ts":"9223372036854775807"} 2024-11-17T21:24:34,304 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T21:24:34,304 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8417a211937373f3c01713928c5cb18b, NAME => 'TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T21:24:34,304 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T21:24:34,305 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731878674304"}]},"ts":"9223372036854775807"} 2024-11-17T21:24:34,308 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T21:24:34,309 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:24:34,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 89 msec 2024-11-17T21:24:34,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-17T21:24:34,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T21:24:34,332 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:34,342 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=240 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;a313eea8709e:46445-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/a313eea8709e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1395276899_22 at /127.0.0.1:59528 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_58963030_22 at /127.0.0.1:59538 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_58963030_22 at /127.0.0.1:35476 [Waiting for operation #381] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1395276899_22 at /127.0.0.1:35340 [Waiting for operation #388] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=439 (was 361) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 12), AvailableMemoryMB=8710 (was 9490) 2024-11-17T21:24:34,350 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=240, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=439, ProcessCount=11, AvailableMemoryMB=8710 2024-11-17T21:24:34,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T21:24:34,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T21:24:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T21:24:34,354 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T21:24:34,354 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:34,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-17T21:24:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T21:24:34,355 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T21:24:34,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741963_1139 (size=959) 2024-11-17T21:24:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T21:24:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T21:24:34,762 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249 2024-11-17T21:24:34,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741964_1140 (size=53) 2024-11-17T21:24:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T21:24:35,171 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:24:35,171 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4554a61bae93fe0dedbab5f4c1513837, disabling compactions & flushes 2024-11-17T21:24:35,171 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:35,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:35,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. after waiting 0 ms 2024-11-17T21:24:35,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:35,172 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:35,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4554a61bae93fe0dedbab5f4c1513837: Waiting for close lock at 1731878675171Disabling compacts and flushes for region at 1731878675171Disabling writes for close at 1731878675172 (+1 ms)Writing region close event to WAL at 1731878675172Closed at 1731878675172 2024-11-17T21:24:35,175 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T21:24:35,175 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731878675175"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731878675175"}]},"ts":"1731878675175"} 2024-11-17T21:24:35,180 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T21:24:35,181 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T21:24:35,181 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878675181"}]},"ts":"1731878675181"} 2024-11-17T21:24:35,183 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T21:24:35,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4554a61bae93fe0dedbab5f4c1513837, ASSIGN}] 2024-11-17T21:24:35,186 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4554a61bae93fe0dedbab5f4c1513837, ASSIGN 2024-11-17T21:24:35,187 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4554a61bae93fe0dedbab5f4c1513837, ASSIGN; state=OFFLINE, location=a313eea8709e,46445,1731878642828; forceNewPlan=false, retain=false 2024-11-17T21:24:35,337 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=4554a61bae93fe0dedbab5f4c1513837, regionState=OPENING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:24:35,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4554a61bae93fe0dedbab5f4c1513837, ASSIGN because future has completed 2024-11-17T21:24:35,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:24:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T21:24:35,495 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:35,496 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} 2024-11-17T21:24:35,496 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:35,496 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:24:35,496 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:35,496 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:35,499 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:35,502 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4554a61bae93fe0dedbab5f4c1513837 columnFamilyName A 2024-11-17T21:24:35,503 DEBUG [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:35,503 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] regionserver.HStore(327): Store=4554a61bae93fe0dedbab5f4c1513837/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:24:35,504 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:35,506 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4554a61bae93fe0dedbab5f4c1513837 columnFamilyName B 2024-11-17T21:24:35,506 DEBUG [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:35,506 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] regionserver.HStore(327): Store=4554a61bae93fe0dedbab5f4c1513837/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:24:35,507 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:35,508 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4554a61bae93fe0dedbab5f4c1513837 columnFamilyName C 2024-11-17T21:24:35,508 DEBUG [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:35,508 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] regionserver.HStore(327): Store=4554a61bae93fe0dedbab5f4c1513837/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:24:35,508 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:35,509 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:35,509 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:35,510 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:35,510 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:35,511 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T21:24:35,512 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:35,514 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T21:24:35,515 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 4554a61bae93fe0dedbab5f4c1513837; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62558763, jitterRate=-0.06780178844928741}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T21:24:35,515 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:35,515 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 4554a61bae93fe0dedbab5f4c1513837: Running coprocessor pre-open hook at 1731878675497Writing region info on filesystem at 1731878675497Initializing all the Stores at 1731878675498 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878675498Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878675498Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878675498Cleaning up temporary data from old regions at 1731878675511 (+13 ms)Running coprocessor post-open hooks at 1731878675515 (+4 ms)Region opened successfully at 1731878675515 2024-11-17T21:24:35,516 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., pid=36, masterSystemTime=1731878675492 2024-11-17T21:24:35,518 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:35,518 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:35,518 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=4554a61bae93fe0dedbab5f4c1513837, regionState=OPEN, openSeqNum=2, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:24:35,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:24:35,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-17T21:24:35,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 in 181 msec 2024-11-17T21:24:35,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-17T21:24:35,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4554a61bae93fe0dedbab5f4c1513837, ASSIGN in 339 msec 2024-11-17T21:24:35,525 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T21:24:35,525 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878675525"}]},"ts":"1731878675525"} 2024-11-17T21:24:35,527 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T21:24:35,528 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T21:24:35,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-17T21:24:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T21:24:36,501 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:36,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b67a313, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:36,503 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:36,503 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:36,505 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@595cff53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:36,507 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a07e7ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:36,508 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:36,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:36,511 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:36,512 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T21:24:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is a313eea8709e,43623,1731878641945 2024-11-17T21:24:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61fade 2024-11-17T21:24:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T21:24:36,515 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T21:24:36,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T21:24:36,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T21:24:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T21:24:36,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741965_1141 (size=995) 2024-11-17T21:24:36,947 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-17T21:24:36,947 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-17T21:24:36,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T21:24:36,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4554a61bae93fe0dedbab5f4c1513837, REOPEN/MOVE}] 2024-11-17T21:24:36,964 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4554a61bae93fe0dedbab5f4c1513837, REOPEN/MOVE 2024-11-17T21:24:36,966 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=4554a61bae93fe0dedbab5f4c1513837, regionState=CLOSING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:24:36,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4554a61bae93fe0dedbab5f4c1513837, REOPEN/MOVE because future has completed 2024-11-17T21:24:36,968 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T21:24:36,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:24:37,121 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,121 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T21:24:37,121 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 4554a61bae93fe0dedbab5f4c1513837, disabling compactions & flushes 2024-11-17T21:24:37,121 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:37,121 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:37,122 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. after waiting 0 ms 2024-11-17T21:24:37,122 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:37,126 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T21:24:37,127 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:37,127 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 4554a61bae93fe0dedbab5f4c1513837: Waiting for close lock at 1731878677121Running coprocessor pre-close hooks at 1731878677121Disabling compacts and flushes for region at 1731878677121Disabling writes for close at 1731878677122 (+1 ms)Writing region close event to WAL at 1731878677122Running coprocessor post-close hooks at 1731878677127 (+5 ms)Closed at 1731878677127 2024-11-17T21:24:37,127 WARN [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 4554a61bae93fe0dedbab5f4c1513837 to self. 2024-11-17T21:24:37,130 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,130 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=4554a61bae93fe0dedbab5f4c1513837, regionState=CLOSED 2024-11-17T21:24:37,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:24:37,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-17T21:24:37,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 in 166 msec 2024-11-17T21:24:37,138 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4554a61bae93fe0dedbab5f4c1513837, REOPEN/MOVE; state=CLOSED, location=a313eea8709e,46445,1731878642828; forceNewPlan=false, retain=true 2024-11-17T21:24:37,290 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=4554a61bae93fe0dedbab5f4c1513837, regionState=OPENING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4554a61bae93fe0dedbab5f4c1513837, REOPEN/MOVE because future has completed 2024-11-17T21:24:37,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:24:37,453 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:37,453 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} 2024-11-17T21:24:37,454 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,454 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:24:37,454 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,454 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,458 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,462 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4554a61bae93fe0dedbab5f4c1513837 columnFamilyName A 2024-11-17T21:24:37,464 DEBUG [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:37,464 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] regionserver.HStore(327): Store=4554a61bae93fe0dedbab5f4c1513837/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:24:37,465 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,465 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4554a61bae93fe0dedbab5f4c1513837 columnFamilyName B 2024-11-17T21:24:37,465 DEBUG [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:37,466 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] regionserver.HStore(327): Store=4554a61bae93fe0dedbab5f4c1513837/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:24:37,466 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,466 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4554a61bae93fe0dedbab5f4c1513837 columnFamilyName C 2024-11-17T21:24:37,466 DEBUG [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:37,467 INFO [StoreOpener-4554a61bae93fe0dedbab5f4c1513837-1 {}] regionserver.HStore(327): Store=4554a61bae93fe0dedbab5f4c1513837/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:24:37,467 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,467 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,468 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,469 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,469 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,470 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T21:24:37,471 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,471 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 4554a61bae93fe0dedbab5f4c1513837; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74680053, jitterRate=0.11281950771808624}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T21:24:37,471 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,472 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 4554a61bae93fe0dedbab5f4c1513837: Running coprocessor pre-open hook at 1731878677454Writing region info on filesystem at 1731878677454Initializing all the Stores at 1731878677455 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878677455Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878677455Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878677455Cleaning up temporary data from old regions at 1731878677469 (+14 ms)Running coprocessor post-open hooks at 1731878677471 (+2 ms)Region opened successfully at 1731878677472 (+1 ms) 2024-11-17T21:24:37,473 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., pid=41, masterSystemTime=1731878677450 2024-11-17T21:24:37,475 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:37,475 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:37,475 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=4554a61bae93fe0dedbab5f4c1513837, regionState=OPEN, openSeqNum=5, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:24:37,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-17T21:24:37,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 in 182 msec 2024-11-17T21:24:37,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-17T21:24:37,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4554a61bae93fe0dedbab5f4c1513837, REOPEN/MOVE in 516 msec 2024-11-17T21:24:37,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-17T21:24:37,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-11-17T21:24:37,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 960 msec 2024-11-17T21:24:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-17T21:24:37,489 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:37,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ffa059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,490 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:37,490 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:37,491 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28c12cdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:37,492 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:37,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1913bfac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59216fed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,494 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:37,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:37,495 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ce03a8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:37,496 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:37,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75fbce1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a1d7814, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:37,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:37,499 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:37,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:37,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:37,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32a5a7b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:37,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:37,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:37,500 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cfdefce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3868a8b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,502 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:37,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:37,503 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c4fc75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:37,504 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:37,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@516dac8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d7d4682, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:37,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:37,506 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d8a8c75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:37,508 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:37,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b351449, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38bb1855, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:37,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:37,511 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4339968f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:37,512 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2851ea9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b3a2f0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,514 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:37,514 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:37,515 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:37,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:37,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:37,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ba2433b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:37,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:37,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:37,516 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:37,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d887f81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5afa5df8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,518 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:37,518 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:37,518 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c39a4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:37,520 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a3b61a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60e92ea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,521 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:24:37,521 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:24:37,522 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:24:37,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:24:37,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:24:37,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6951cdd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:24:37,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:24:37,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:37,523 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:24:37,524 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594dbf65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:24:37,525 DEBUG [Thread-696 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:37,525 DEBUG [Thread-702 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:37,525 DEBUG [Thread-713 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:37,525 DEBUG [Thread-698 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:37,525 DEBUG [Thread-707 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:37,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:37,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:37,526 DEBUG [Thread-696 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:37,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:37,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:37,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:37,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-17T21:24:37,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:37,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:37,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T21:24:37,528 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:37,528 DEBUG [Thread-696 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:37,528 DEBUG [Thread-711 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:37,528 DEBUG [Thread-700 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:37,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:37,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:37,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:37,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:37,529 DEBUG [Thread-704 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:37,530 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:37,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:37,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:37,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:37,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:37,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:37,530 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:37,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:37,531 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:37,531 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:37,531 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:37,533 DEBUG [Thread-709 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:24:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:24:37,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:24:37,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:24:37,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:24:37,537 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:24:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:24:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:24:37,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:24:37,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:24:37,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:24:37,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:24:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:24:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,550 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:37,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T21:24:37,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:37,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878737587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:37,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878737589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878737590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:37,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a6b83f50602447a39ee9bc9a9f19833a_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878677542/Put/seqid=0 2024-11-17T21:24:37,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878737596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:37,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878737597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:37,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741966_1142 (size=17034) 2024-11-17T21:24:37,616 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:37,621 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a6b83f50602447a39ee9bc9a9f19833a_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a6b83f50602447a39ee9bc9a9f19833a_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,622 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/a2c0426fec834e3cb85b08e11e805513, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T21:24:37,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/a2c0426fec834e3cb85b08e11e805513 is 175, key is test_row_0/A:col10/1731878677542/Put/seqid=0 2024-11-17T21:24:37,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741967_1143 (size=48139) 2024-11-17T21:24:37,654 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/a2c0426fec834e3cb85b08e11e805513 2024-11-17T21:24:37,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T21:24:37,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:37,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:37,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/3617bdf2d7c24a05870219150a8fe0f4 is 50, key is test_row_0/B:col10/1731878677542/Put/seqid=0 2024-11-17T21:24:37,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:37,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878737701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:37,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878737701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:37,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:37,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878737701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:37,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878737712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878737712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:37,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741968_1144 (size=12001) 2024-11-17T21:24:37,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/3617bdf2d7c24a05870219150a8fe0f4 2024-11-17T21:24:37,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/e215cfb06dc742d68badc5e39e4360c5 is 50, key is test_row_0/C:col10/1731878677542/Put/seqid=0 2024-11-17T21:24:37,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741969_1145 (size=12001) 2024-11-17T21:24:37,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/e215cfb06dc742d68badc5e39e4360c5 2024-11-17T21:24:37,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/a2c0426fec834e3cb85b08e11e805513 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/a2c0426fec834e3cb85b08e11e805513 2024-11-17T21:24:37,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/a2c0426fec834e3cb85b08e11e805513, entries=250, sequenceid=16, filesize=47.0 K 2024-11-17T21:24:37,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/3617bdf2d7c24a05870219150a8fe0f4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3617bdf2d7c24a05870219150a8fe0f4 2024-11-17T21:24:37,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3617bdf2d7c24a05870219150a8fe0f4, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T21:24:37,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/e215cfb06dc742d68badc5e39e4360c5 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/e215cfb06dc742d68badc5e39e4360c5 2024-11-17T21:24:37,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/e215cfb06dc742d68badc5e39e4360c5, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T21:24:37,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4554a61bae93fe0dedbab5f4c1513837 in 281ms, sequenceid=16, compaction requested=false 2024-11-17T21:24:37,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T21:24:37,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T21:24:37,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:37,890 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T21:24:37,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111769c1b0c0201349179858a048035bc797_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878677593/Put/seqid=0 2024-11-17T21:24:37,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878737930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878737931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:37,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878737933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:37,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741970_1146 (size=12154) 2024-11-17T21:24:37,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:37,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:37,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878737940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878737938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:37,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:37,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:37,962 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111769c1b0c0201349179858a048035bc797_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111769c1b0c0201349179858a048035bc797_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:37,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/ea476ef4e55b4bec9cd1e1ee7971555b, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:37,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/ea476ef4e55b4bec9cd1e1ee7971555b is 175, key is test_row_0/A:col10/1731878677593/Put/seqid=0 2024-11-17T21:24:37,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741971_1147 (size=30955) 2024-11-17T21:24:37,996 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/ea476ef4e55b4bec9cd1e1ee7971555b 2024-11-17T21:24:38,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/08bb1d009b3a429b87988616bebe5307 is 50, key is test_row_0/B:col10/1731878677593/Put/seqid=0 2024-11-17T21:24:38,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741972_1148 (size=12001) 2024-11-17T21:24:38,050 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/08bb1d009b3a429b87988616bebe5307 2024-11-17T21:24:38,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878738052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878738052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878738052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878738061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878738061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/6c57f426be0d429f8727d4fd1d770500 is 50, key is test_row_0/C:col10/1731878677593/Put/seqid=0 2024-11-17T21:24:38,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741973_1149 (size=12001) 2024-11-17T21:24:38,110 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/6c57f426be0d429f8727d4fd1d770500 2024-11-17T21:24:38,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/ea476ef4e55b4bec9cd1e1ee7971555b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/ea476ef4e55b4bec9cd1e1ee7971555b 2024-11-17T21:24:38,140 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/ea476ef4e55b4bec9cd1e1ee7971555b, entries=150, sequenceid=40, filesize=30.2 K 2024-11-17T21:24:38,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/08bb1d009b3a429b87988616bebe5307 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/08bb1d009b3a429b87988616bebe5307 2024-11-17T21:24:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T21:24:38,160 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/08bb1d009b3a429b87988616bebe5307, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T21:24:38,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/6c57f426be0d429f8727d4fd1d770500 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/6c57f426be0d429f8727d4fd1d770500 2024-11-17T21:24:38,168 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/6c57f426be0d429f8727d4fd1d770500, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T21:24:38,169 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4554a61bae93fe0dedbab5f4c1513837 in 278ms, sequenceid=40, compaction requested=false 2024-11-17T21:24:38,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:38,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:38,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-17T21:24:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-17T21:24:38,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-17T21:24:38,197 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 663 msec 2024-11-17T21:24:38,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 671 msec 2024-11-17T21:24:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:38,274 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:38,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T21:24:38,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b29e568de9774c42a3cd96d0b19964df_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878677937/Put/seqid=0 2024-11-17T21:24:38,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878738315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878738319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878738321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878738323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878738323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741974_1150 (size=14594) 2024-11-17T21:24:38,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878738431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878738442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878738442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878738442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878738442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878738642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878738651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878738651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878738651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878738651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T21:24:38,661 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:38,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-17T21:24:38,664 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T21:24:38,665 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:38,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:38,734 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:38,740 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b29e568de9774c42a3cd96d0b19964df_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b29e568de9774c42a3cd96d0b19964df_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:38,742 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/c6e0665b8b634e368ad5ea0f1910fe9d, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:38,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/c6e0665b8b634e368ad5ea0f1910fe9d is 175, key is test_row_0/A:col10/1731878677937/Put/seqid=0 2024-11-17T21:24:38,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741975_1151 (size=39549) 2024-11-17T21:24:38,762 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/c6e0665b8b634e368ad5ea0f1910fe9d 2024-11-17T21:24:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T21:24:38,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/9f9a2f2513c948eba4f4cd0ae766ace3 is 50, key is test_row_0/B:col10/1731878677937/Put/seqid=0 2024-11-17T21:24:38,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741976_1152 (size=12001) 2024-11-17T21:24:38,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/9f9a2f2513c948eba4f4cd0ae766ace3 2024-11-17T21:24:38,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T21:24:38,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:38,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:38,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:38,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:38,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:38,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/2c6e80e8862b493198ae90f31c5221c9 is 50, key is test_row_0/C:col10/1731878677937/Put/seqid=0 2024-11-17T21:24:38,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741977_1153 (size=12001) 2024-11-17T21:24:38,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878738951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878738961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878738961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878738961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878738961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:38,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T21:24:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:38,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T21:24:39,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T21:24:39,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:39,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:39,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:39,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:39,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:39,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/2c6e80e8862b493198ae90f31c5221c9 2024-11-17T21:24:39,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/c6e0665b8b634e368ad5ea0f1910fe9d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/c6e0665b8b634e368ad5ea0f1910fe9d 2024-11-17T21:24:39,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/c6e0665b8b634e368ad5ea0f1910fe9d, entries=200, sequenceid=53, filesize=38.6 K 2024-11-17T21:24:39,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/9f9a2f2513c948eba4f4cd0ae766ace3 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/9f9a2f2513c948eba4f4cd0ae766ace3 2024-11-17T21:24:39,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/9f9a2f2513c948eba4f4cd0ae766ace3, entries=150, sequenceid=53, filesize=11.7 K 2024-11-17T21:24:39,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T21:24:39,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:39,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/2c6e80e8862b493198ae90f31c5221c9 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/2c6e80e8862b493198ae90f31c5221c9 2024-11-17T21:24:39,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:39,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:39,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:39,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T21:24:39,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/2c6e80e8862b493198ae90f31c5221c9, entries=150, sequenceid=53, filesize=11.7 K 2024-11-17T21:24:39,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 4554a61bae93fe0dedbab5f4c1513837 in 1019ms, sequenceid=53, compaction requested=true 2024-11-17T21:24:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:39,294 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:39,294 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:39,296 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:39,296 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:39,296 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/B is initiating minor compaction (all files) 2024-11-17T21:24:39,296 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/A is initiating minor compaction (all files) 2024-11-17T21:24:39,296 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/B in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:39,296 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/A in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:39,296 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3617bdf2d7c24a05870219150a8fe0f4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/08bb1d009b3a429b87988616bebe5307, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/9f9a2f2513c948eba4f4cd0ae766ace3] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=35.2 K 2024-11-17T21:24:39,296 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/a2c0426fec834e3cb85b08e11e805513, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/ea476ef4e55b4bec9cd1e1ee7971555b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/c6e0665b8b634e368ad5ea0f1910fe9d] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=115.9 K 2024-11-17T21:24:39,297 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:39,297 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/a2c0426fec834e3cb85b08e11e805513, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/ea476ef4e55b4bec9cd1e1ee7971555b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/c6e0665b8b634e368ad5ea0f1910fe9d] 2024-11-17T21:24:39,297 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 3617bdf2d7c24a05870219150a8fe0f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731878677542 2024-11-17T21:24:39,297 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2c0426fec834e3cb85b08e11e805513, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731878677542 2024-11-17T21:24:39,298 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 08bb1d009b3a429b87988616bebe5307, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731878677585 2024-11-17T21:24:39,298 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea476ef4e55b4bec9cd1e1ee7971555b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731878677585 2024-11-17T21:24:39,298 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f9a2f2513c948eba4f4cd0ae766ace3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731878677930 2024-11-17T21:24:39,298 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6e0665b8b634e368ad5ea0f1910fe9d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731878677923 2024-11-17T21:24:39,306 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:39,312 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#B#compaction#136 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:39,312 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/1594dc93447f4c9097fffd2f6e2b3d44 is 50, key is test_row_0/B:col10/1731878677937/Put/seqid=0 2024-11-17T21:24:39,325 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111710688bbca4a64ea7b50f2968dbe71fe9_4554a61bae93fe0dedbab5f4c1513837 store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:39,329 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111710688bbca4a64ea7b50f2968dbe71fe9_4554a61bae93fe0dedbab5f4c1513837, store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:39,329 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111710688bbca4a64ea7b50f2968dbe71fe9_4554a61bae93fe0dedbab5f4c1513837 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:39,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741978_1154 (size=12104) 2024-11-17T21:24:39,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741979_1155 (size=4469) 2024-11-17T21:24:39,354 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#A#compaction#135 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:39,355 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/f4f9b441b6604f57a1a9b6fa132f4159 is 175, key is test_row_0/A:col10/1731878677937/Put/seqid=0 2024-11-17T21:24:39,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741980_1156 (size=31058) 2024-11-17T21:24:39,374 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/f4f9b441b6604f57a1a9b6fa132f4159 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/f4f9b441b6604f57a1a9b6fa132f4159 2024-11-17T21:24:39,381 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/A of 4554a61bae93fe0dedbab5f4c1513837 into f4f9b441b6604f57a1a9b6fa132f4159(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:39,381 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:39,381 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/A, priority=13, startTime=1731878679294; duration=0sec 2024-11-17T21:24:39,381 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:39,381 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:A 2024-11-17T21:24:39,381 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:39,385 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:39,385 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/C is initiating minor compaction (all files) 2024-11-17T21:24:39,385 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/C in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:39,385 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/e215cfb06dc742d68badc5e39e4360c5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/6c57f426be0d429f8727d4fd1d770500, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/2c6e80e8862b493198ae90f31c5221c9] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=35.2 K 2024-11-17T21:24:39,385 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting e215cfb06dc742d68badc5e39e4360c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731878677542 2024-11-17T21:24:39,386 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c57f426be0d429f8727d4fd1d770500, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731878677585 2024-11-17T21:24:39,386 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c6e80e8862b493198ae90f31c5221c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731878677930 2024-11-17T21:24:39,394 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#C#compaction#137 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:39,396 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/084d4f56025444759de9297fd73e13de is 50, key is test_row_0/C:col10/1731878677937/Put/seqid=0 2024-11-17T21:24:39,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741981_1157 (size=12104) 2024-11-17T21:24:39,423 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/084d4f56025444759de9297fd73e13de as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/084d4f56025444759de9297fd73e13de 2024-11-17T21:24:39,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T21:24:39,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:39,436 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T21:24:39,453 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/C of 4554a61bae93fe0dedbab5f4c1513837 into 084d4f56025444759de9297fd73e13de(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:39,453 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:39,453 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/C, priority=13, startTime=1731878679294; duration=0sec 2024-11-17T21:24:39,453 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:39,453 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:C 2024-11-17T21:24:39,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:39,476 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T21:24:39,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170e86d88a78ec498abe7902c459402ee3_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878678320/Put/seqid=0 2024-11-17T21:24:39,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878739488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:39,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878739489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:39,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878739501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:39,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878739501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:39,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:39,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878739501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:39,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:39,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741982_1158 (size=12154) 2024-11-17T21:24:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:39,533 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170e86d88a78ec498abe7902c459402ee3_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170e86d88a78ec498abe7902c459402ee3_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:39,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/78c07c28cef64c86abfad52ab26b787d, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:39,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/78c07c28cef64c86abfad52ab26b787d is 175, key is test_row_0/A:col10/1731878678320/Put/seqid=0 2024-11-17T21:24:39,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741983_1159 (size=30955) 2024-11-17T21:24:39,554 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/78c07c28cef64c86abfad52ab26b787d 2024-11-17T21:24:39,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/3e99d0c1307f4b2a92e0c912e099df64 is 50, key is test_row_0/B:col10/1731878678320/Put/seqid=0 2024-11-17T21:24:39,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741984_1160 (size=12001) 2024-11-17T21:24:39,584 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/3e99d0c1307f4b2a92e0c912e099df64 2024-11-17T21:24:39,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/55d929599f2e4600934a9b38da334272 is 50, key is test_row_0/C:col10/1731878678320/Put/seqid=0 2024-11-17T21:24:39,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878739612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:39,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878739611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:39,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878739614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:39,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:39,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878739614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:39,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:39,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:39,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878739633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:39,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741985_1161 (size=12001) 2024-11-17T21:24:39,747 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/1594dc93447f4c9097fffd2f6e2b3d44 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/1594dc93447f4c9097fffd2f6e2b3d44 2024-11-17T21:24:39,754 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/B of 4554a61bae93fe0dedbab5f4c1513837 into 1594dc93447f4c9097fffd2f6e2b3d44(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:39,755 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:39,755 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/B, priority=13, startTime=1731878679294; duration=0sec 2024-11-17T21:24:39,755 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:39,755 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:B 2024-11-17T21:24:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T21:24:39,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878739841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:39,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878739841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:39,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878739851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:39,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878739851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:39,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878739851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,067 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/55d929599f2e4600934a9b38da334272 2024-11-17T21:24:40,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/78c07c28cef64c86abfad52ab26b787d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/78c07c28cef64c86abfad52ab26b787d 2024-11-17T21:24:40,098 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/78c07c28cef64c86abfad52ab26b787d, entries=150, sequenceid=79, filesize=30.2 K 2024-11-17T21:24:40,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/3e99d0c1307f4b2a92e0c912e099df64 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3e99d0c1307f4b2a92e0c912e099df64 2024-11-17T21:24:40,106 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3e99d0c1307f4b2a92e0c912e099df64, entries=150, sequenceid=79, filesize=11.7 K 2024-11-17T21:24:40,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/55d929599f2e4600934a9b38da334272 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/55d929599f2e4600934a9b38da334272 2024-11-17T21:24:40,117 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/55d929599f2e4600934a9b38da334272, entries=150, sequenceid=79, filesize=11.7 K 2024-11-17T21:24:40,118 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for 4554a61bae93fe0dedbab5f4c1513837 in 682ms, sequenceid=79, compaction requested=false 2024-11-17T21:24:40,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:40,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:40,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-17T21:24:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-17T21:24:40,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-17T21:24:40,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4600 sec 2024-11-17T21:24:40,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4680 sec 2024-11-17T21:24:40,155 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:40,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T21:24:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:40,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b5eba6729a914011b92c7a332fad70eb_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878679494/Put/seqid=0 2024-11-17T21:24:40,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878740230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878740233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878740235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878740236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741986_1162 (size=14594) 2024-11-17T21:24:40,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878740245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878740351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878740351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878740351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878740352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878740361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878740571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878740571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878740572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878740572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878740581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,646 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:40,652 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b5eba6729a914011b92c7a332fad70eb_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b5eba6729a914011b92c7a332fad70eb_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:40,653 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/3017606f80b34a74b393ce3ea1838fe5, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:40,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/3017606f80b34a74b393ce3ea1838fe5 is 175, key is test_row_0/A:col10/1731878679494/Put/seqid=0 2024-11-17T21:24:40,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741987_1163 (size=39549) 2024-11-17T21:24:40,689 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/3017606f80b34a74b393ce3ea1838fe5 2024-11-17T21:24:40,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/93037c3849ed45c1825538aee7307028 is 50, key is test_row_0/B:col10/1731878679494/Put/seqid=0 2024-11-17T21:24:40,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741988_1164 (size=12001) 2024-11-17T21:24:40,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T21:24:40,811 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:40,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-17T21:24:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T21:24:40,813 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:40,814 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:40,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:40,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878740882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878740882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878740882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878740882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:40,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878740892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T21:24:40,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T21:24:40,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:40,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:40,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:40,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:40,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:40,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:41,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T21:24:41,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:41,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T21:24:41,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/93037c3849ed45c1825538aee7307028 2024-11-17T21:24:41,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/4797818bef6d4ae69b0edac9e459a54d is 50, key is test_row_0/C:col10/1731878679494/Put/seqid=0 2024-11-17T21:24:41,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741989_1165 (size=12001) 2024-11-17T21:24:41,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T21:24:41,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:41,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:41,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:41,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:41,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:41,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878741391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:41,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878741391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:41,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878741391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:41,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878741391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:41,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878741402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:41,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T21:24:41,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:41,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:41,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:41,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:41,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T21:24:41,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/4797818bef6d4ae69b0edac9e459a54d 2024-11-17T21:24:41,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T21:24:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:41,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:41,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/3017606f80b34a74b393ce3ea1838fe5 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3017606f80b34a74b393ce3ea1838fe5 2024-11-17T21:24:41,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3017606f80b34a74b393ce3ea1838fe5, entries=200, sequenceid=95, filesize=38.6 K 2024-11-17T21:24:41,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/93037c3849ed45c1825538aee7307028 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/93037c3849ed45c1825538aee7307028 2024-11-17T21:24:41,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/93037c3849ed45c1825538aee7307028, entries=150, sequenceid=95, filesize=11.7 K 2024-11-17T21:24:41,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/4797818bef6d4ae69b0edac9e459a54d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/4797818bef6d4ae69b0edac9e459a54d 2024-11-17T21:24:41,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/4797818bef6d4ae69b0edac9e459a54d, entries=150, sequenceid=95, filesize=11.7 K 2024-11-17T21:24:41,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 4554a61bae93fe0dedbab5f4c1513837 in 1459ms, sequenceid=95, compaction requested=true 2024-11-17T21:24:41,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:41,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:41,614 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:41,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:41,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:41,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:41,614 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:41,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:41,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:41,615 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:41,615 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:41,615 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/B is initiating minor compaction (all files) 2024-11-17T21:24:41,615 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/A is initiating minor compaction (all files) 2024-11-17T21:24:41,615 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/B in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:41,615 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/A in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:41,615 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/1594dc93447f4c9097fffd2f6e2b3d44, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3e99d0c1307f4b2a92e0c912e099df64, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/93037c3849ed45c1825538aee7307028] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=35.3 K 2024-11-17T21:24:41,615 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/f4f9b441b6604f57a1a9b6fa132f4159, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/78c07c28cef64c86abfad52ab26b787d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3017606f80b34a74b393ce3ea1838fe5] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=99.2 K 2024-11-17T21:24:41,616 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:41,616 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/f4f9b441b6604f57a1a9b6fa132f4159, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/78c07c28cef64c86abfad52ab26b787d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3017606f80b34a74b393ce3ea1838fe5] 2024-11-17T21:24:41,616 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 1594dc93447f4c9097fffd2f6e2b3d44, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731878677930 2024-11-17T21:24:41,616 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4f9b441b6604f57a1a9b6fa132f4159, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731878677930 2024-11-17T21:24:41,616 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e99d0c1307f4b2a92e0c912e099df64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731878678307 2024-11-17T21:24:41,616 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78c07c28cef64c86abfad52ab26b787d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731878678307 2024-11-17T21:24:41,617 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 93037c3849ed45c1825538aee7307028, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731878679494 2024-11-17T21:24:41,617 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3017606f80b34a74b393ce3ea1838fe5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731878679487 2024-11-17T21:24:41,624 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:41,625 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#B#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:41,625 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/b7316dfb45e646709ddfc360ef480548 is 50, key is test_row_0/B:col10/1731878679494/Put/seqid=0 2024-11-17T21:24:41,626 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411178c434d64b7e7405bac5315021cbdbb4d_4554a61bae93fe0dedbab5f4c1513837 store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:41,630 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411178c434d64b7e7405bac5315021cbdbb4d_4554a61bae93fe0dedbab5f4c1513837, store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:41,631 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178c434d64b7e7405bac5315021cbdbb4d_4554a61bae93fe0dedbab5f4c1513837 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:41,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741991_1167 (size=4469) 2024-11-17T21:24:41,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741990_1166 (size=12207) 2024-11-17T21:24:41,645 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/b7316dfb45e646709ddfc360ef480548 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/b7316dfb45e646709ddfc360ef480548 2024-11-17T21:24:41,651 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/B of 4554a61bae93fe0dedbab5f4c1513837 into b7316dfb45e646709ddfc360ef480548(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:41,651 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:41,651 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/B, priority=13, startTime=1731878681614; duration=0sec 2024-11-17T21:24:41,651 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:41,651 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:B 2024-11-17T21:24:41,651 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:41,652 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:41,652 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/C is initiating minor compaction (all files) 2024-11-17T21:24:41,653 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/C in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:41,653 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/084d4f56025444759de9297fd73e13de, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/55d929599f2e4600934a9b38da334272, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/4797818bef6d4ae69b0edac9e459a54d] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=35.3 K 2024-11-17T21:24:41,653 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 084d4f56025444759de9297fd73e13de, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731878677930 2024-11-17T21:24:41,653 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 55d929599f2e4600934a9b38da334272, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731878678307 2024-11-17T21:24:41,655 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 4797818bef6d4ae69b0edac9e459a54d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731878679494 2024-11-17T21:24:41,663 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#C#compaction#146 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:41,663 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/d250871dc88a4e238270eb6f3d6d5f8d is 50, key is test_row_0/C:col10/1731878679494/Put/seqid=0 2024-11-17T21:24:41,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741992_1168 (size=12207) 2024-11-17T21:24:41,679 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/d250871dc88a4e238270eb6f3d6d5f8d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/d250871dc88a4e238270eb6f3d6d5f8d 2024-11-17T21:24:41,684 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/C of 4554a61bae93fe0dedbab5f4c1513837 into d250871dc88a4e238270eb6f3d6d5f8d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:41,684 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:41,684 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/C, priority=13, startTime=1731878681614; duration=0sec 2024-11-17T21:24:41,685 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:41,685 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:C 2024-11-17T21:24:41,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T21:24:41,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:41,738 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T21:24:41,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b8f27a56ff954dc68332759e7f4f4c0f_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878680234/Put/seqid=0 2024-11-17T21:24:41,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741993_1169 (size=12154) 2024-11-17T21:24:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T21:24:42,038 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#A#compaction#145 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:42,039 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/9fe2e7d33cdd4d3da2c1762468b9dc87 is 175, key is test_row_0/A:col10/1731878679494/Put/seqid=0 2024-11-17T21:24:42,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741994_1170 (size=31161) 2024-11-17T21:24:42,090 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/9fe2e7d33cdd4d3da2c1762468b9dc87 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9fe2e7d33cdd4d3da2c1762468b9dc87 2024-11-17T21:24:42,098 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/A of 4554a61bae93fe0dedbab5f4c1513837 into 9fe2e7d33cdd4d3da2c1762468b9dc87(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:42,098 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:42,098 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/A, priority=13, startTime=1731878681614; duration=0sec 2024-11-17T21:24:42,098 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:42,098 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:A 2024-11-17T21:24:42,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,161 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b8f27a56ff954dc68332759e7f4f4c0f_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b8f27a56ff954dc68332759e7f4f4c0f_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:42,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/f470c50971064114bc6ad21390cca78a, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/f470c50971064114bc6ad21390cca78a is 175, key is test_row_0/A:col10/1731878680234/Put/seqid=0 2024-11-17T21:24:42,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741995_1171 (size=30955) 2024-11-17T21:24:42,194 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/f470c50971064114bc6ad21390cca78a 2024-11-17T21:24:42,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/07a7cff37379463c83e5436e3452ac5d is 50, key is test_row_0/B:col10/1731878680234/Put/seqid=0 2024-11-17T21:24:42,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741996_1172 (size=12001) 2024-11-17T21:24:42,241 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/07a7cff37379463c83e5436e3452ac5d 2024-11-17T21:24:42,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/7ef3711f215740c7aa1902442da98439 is 50, key is test_row_0/C:col10/1731878680234/Put/seqid=0 2024-11-17T21:24:42,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741997_1173 (size=12001) 2024-11-17T21:24:42,285 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/7ef3711f215740c7aa1902442da98439 2024-11-17T21:24:42,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/f470c50971064114bc6ad21390cca78a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/f470c50971064114bc6ad21390cca78a 2024-11-17T21:24:42,297 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/f470c50971064114bc6ad21390cca78a, entries=150, sequenceid=119, filesize=30.2 K 2024-11-17T21:24:42,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/07a7cff37379463c83e5436e3452ac5d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/07a7cff37379463c83e5436e3452ac5d 2024-11-17T21:24:42,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,309 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/07a7cff37379463c83e5436e3452ac5d, entries=150, sequenceid=119, filesize=11.7 K 2024-11-17T21:24:42,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/7ef3711f215740c7aa1902442da98439 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/7ef3711f215740c7aa1902442da98439 2024-11-17T21:24:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,317 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/7ef3711f215740c7aa1902442da98439, entries=150, sequenceid=119, filesize=11.7 K 2024-11-17T21:24:42,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,319 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 4554a61bae93fe0dedbab5f4c1513837 in 580ms, sequenceid=119, compaction requested=false 2024-11-17T21:24:42,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:42,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:42,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-17T21:24:42,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-17T21:24:42,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-17T21:24:42,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5060 sec 2024-11-17T21:24:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.5120 sec 2024-11-17T21:24:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:42,526 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:42,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:24:42,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f15a4a6c79064e71b82eed19959c22ba_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878682525/Put/seqid=0 2024-11-17T21:24:42,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741998_1174 (size=14744) 2024-11-17T21:24:42,566 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:42,574 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f15a4a6c79064e71b82eed19959c22ba_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f15a4a6c79064e71b82eed19959c22ba_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:42,575 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/9b4f34b28f8c43eda56e87644431f2c3, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:42,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/9b4f34b28f8c43eda56e87644431f2c3 is 175, key is test_row_0/A:col10/1731878682525/Put/seqid=0 2024-11-17T21:24:42,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741999_1175 (size=39699) 2024-11-17T21:24:42,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878742625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:42,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878742633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:42,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878742635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:42,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:42,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878742638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:42,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:42,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:42,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878742638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:42,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:42,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878742741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:42,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878742751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:42,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878742751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:42,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878742751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:42,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878742751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:42,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878742951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T21:24:42,961 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:42,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-17T21:24:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T21:24:42,964 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:42,965 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:42,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:42,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878742971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:42,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878742972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:42,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878742972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:42,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878742972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:42,986 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/9b4f34b28f8c43eda56e87644431f2c3 2024-11-17T21:24:43,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/51f31251b4614243844de41c49c34b7e is 50, key is test_row_0/B:col10/1731878682525/Put/seqid=0 2024-11-17T21:24:43,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742000_1176 (size=12151) 2024-11-17T21:24:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T21:24:43,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T21:24:43,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:43,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:43,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:43,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878743260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:43,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T21:24:43,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:43,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:43,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:43,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T21:24:43,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878743281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:43,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878743281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:43,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878743282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878743282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:43,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T21:24:43,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:43,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:43,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:43,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/51f31251b4614243844de41c49c34b7e 2024-11-17T21:24:43,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/414e62a1b04a43458378304eb7bc604c is 50, key is test_row_0/C:col10/1731878682525/Put/seqid=0 2024-11-17T21:24:43,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742001_1177 (size=12151) 2024-11-17T21:24:43,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T21:24:43,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:43,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:43,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:43,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T21:24:43,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T21:24:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:43,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878743771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:43,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878743791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878743791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:43,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878743791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878743791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:43,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:43,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/414e62a1b04a43458378304eb7bc604c 2024-11-17T21:24:43,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/9b4f34b28f8c43eda56e87644431f2c3 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9b4f34b28f8c43eda56e87644431f2c3 2024-11-17T21:24:43,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9b4f34b28f8c43eda56e87644431f2c3, entries=200, sequenceid=132, filesize=38.8 K 2024-11-17T21:24:43,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T21:24:43,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:43,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:43,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:43,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/51f31251b4614243844de41c49c34b7e as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/51f31251b4614243844de41c49c34b7e 2024-11-17T21:24:43,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:43,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/51f31251b4614243844de41c49c34b7e, entries=150, sequenceid=132, filesize=11.9 K 2024-11-17T21:24:43,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/414e62a1b04a43458378304eb7bc604c as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/414e62a1b04a43458378304eb7bc604c 2024-11-17T21:24:43,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/414e62a1b04a43458378304eb7bc604c, entries=150, sequenceid=132, filesize=11.9 K 2024-11-17T21:24:43,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for 4554a61bae93fe0dedbab5f4c1513837 in 1379ms, sequenceid=132, compaction requested=true 2024-11-17T21:24:43,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:43,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:43,905 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:43,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:43,905 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:43,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:43,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:43,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:43,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:43,907 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:43,907 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:43,907 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/A is initiating minor compaction (all files) 2024-11-17T21:24:43,907 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/B is initiating minor compaction (all files) 2024-11-17T21:24:43,907 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/B in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:43,907 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/A in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:43,907 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/b7316dfb45e646709ddfc360ef480548, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/07a7cff37379463c83e5436e3452ac5d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/51f31251b4614243844de41c49c34b7e] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=35.5 K 2024-11-17T21:24:43,907 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9fe2e7d33cdd4d3da2c1762468b9dc87, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/f470c50971064114bc6ad21390cca78a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9b4f34b28f8c43eda56e87644431f2c3] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=99.4 K 2024-11-17T21:24:43,907 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:43,907 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9fe2e7d33cdd4d3da2c1762468b9dc87, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/f470c50971064114bc6ad21390cca78a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9b4f34b28f8c43eda56e87644431f2c3] 2024-11-17T21:24:43,907 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting b7316dfb45e646709ddfc360ef480548, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731878679494 2024-11-17T21:24:43,907 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fe2e7d33cdd4d3da2c1762468b9dc87, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731878679494 2024-11-17T21:24:43,908 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 07a7cff37379463c83e5436e3452ac5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731878680232 2024-11-17T21:24:43,908 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting f470c50971064114bc6ad21390cca78a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731878680232 2024-11-17T21:24:43,908 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 51f31251b4614243844de41c49c34b7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731878682519 2024-11-17T21:24:43,908 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b4f34b28f8c43eda56e87644431f2c3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731878682511 2024-11-17T21:24:43,913 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:43,915 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#B#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:43,915 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117e8795d1d05c34157bab2b763b8fc2c25_4554a61bae93fe0dedbab5f4c1513837 store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:43,916 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/0c05cf32726c4422a86ddeab34ae6717 is 50, key is test_row_0/B:col10/1731878682525/Put/seqid=0 2024-11-17T21:24:43,917 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117e8795d1d05c34157bab2b763b8fc2c25_4554a61bae93fe0dedbab5f4c1513837, store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:43,918 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e8795d1d05c34157bab2b763b8fc2c25_4554a61bae93fe0dedbab5f4c1513837 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:43,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742003_1179 (size=4469) 2024-11-17T21:24:43,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742002_1178 (size=12459) 2024-11-17T21:24:44,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T21:24:44,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:44,046 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T21:24:44,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a622a227d0624585bc253bda95d1f1ed_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878682616/Put/seqid=0 2024-11-17T21:24:44,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742004_1180 (size=12304) 2024-11-17T21:24:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T21:24:44,368 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#A#compaction#153 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:44,369 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/75d3585afa3e42c3a8423a73d37d0536 is 175, key is test_row_0/A:col10/1731878682525/Put/seqid=0 2024-11-17T21:24:44,384 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/0c05cf32726c4422a86ddeab34ae6717 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/0c05cf32726c4422a86ddeab34ae6717 2024-11-17T21:24:44,391 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/B of 4554a61bae93fe0dedbab5f4c1513837 into 0c05cf32726c4422a86ddeab34ae6717(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:44,391 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:44,391 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/B, priority=13, startTime=1731878683905; duration=0sec 2024-11-17T21:24:44,391 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:44,391 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:B 2024-11-17T21:24:44,391 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:44,392 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:44,392 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/C is initiating minor compaction (all files) 2024-11-17T21:24:44,392 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/C in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:44,393 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/d250871dc88a4e238270eb6f3d6d5f8d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/7ef3711f215740c7aa1902442da98439, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/414e62a1b04a43458378304eb7bc604c] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=35.5 K 2024-11-17T21:24:44,393 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d250871dc88a4e238270eb6f3d6d5f8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731878679494 2024-11-17T21:24:44,396 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ef3711f215740c7aa1902442da98439, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731878680232 2024-11-17T21:24:44,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742005_1181 (size=31413) 2024-11-17T21:24:44,398 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 414e62a1b04a43458378304eb7bc604c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731878682519 2024-11-17T21:24:44,418 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#C#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:44,419 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/67e313311bdb46b7b5450c13d5d46181 is 50, key is test_row_0/C:col10/1731878682525/Put/seqid=0 2024-11-17T21:24:44,431 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/75d3585afa3e42c3a8423a73d37d0536 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/75d3585afa3e42c3a8423a73d37d0536 2024-11-17T21:24:44,443 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/A of 4554a61bae93fe0dedbab5f4c1513837 into 75d3585afa3e42c3a8423a73d37d0536(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:44,444 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:44,444 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/A, priority=13, startTime=1731878683905; duration=0sec 2024-11-17T21:24:44,444 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:44,444 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:A 2024-11-17T21:24:44,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:44,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742006_1182 (size=12459) 2024-11-17T21:24:44,465 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a622a227d0624585bc253bda95d1f1ed_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a622a227d0624585bc253bda95d1f1ed_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:44,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/999fe71bd900436d8392e09e1034020d, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:44,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/999fe71bd900436d8392e09e1034020d is 175, key is test_row_0/A:col10/1731878682616/Put/seqid=0 2024-11-17T21:24:44,475 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/67e313311bdb46b7b5450c13d5d46181 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/67e313311bdb46b7b5450c13d5d46181 2024-11-17T21:24:44,491 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/C of 4554a61bae93fe0dedbab5f4c1513837 into 67e313311bdb46b7b5450c13d5d46181(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:44,492 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:44,492 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/C, priority=13, startTime=1731878683906; duration=0sec 2024-11-17T21:24:44,492 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:44,492 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:C 2024-11-17T21:24:44,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742007_1183 (size=31105) 2024-11-17T21:24:44,507 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/999fe71bd900436d8392e09e1034020d 2024-11-17T21:24:44,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/92bd1421082a42d8a121bc4fe3455617 is 50, key is test_row_0/B:col10/1731878682616/Put/seqid=0 2024-11-17T21:24:44,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742008_1184 (size=12151) 2024-11-17T21:24:44,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:44,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:44,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878744813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878744811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:44,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878744814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:44,817 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_0' in 4554a61bae93fe0dedbab5f4c1513837 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:24:44,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:44,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:44,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878744815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878744816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:44,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:44,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878744921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:44,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:44,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878744921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:44,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878744930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:44,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878744931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:44,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:44,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:44,935 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/92bd1421082a42d8a121bc4fe3455617 2024-11-17T21:24:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/ab6759bb8fb347449d28e0d3cd585ac1 is 50, key is test_row_0/C:col10/1731878682616/Put/seqid=0 2024-11-17T21:24:44,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742009_1185 (size=12151) 2024-11-17T21:24:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T21:24:45,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878745131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:45,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878745132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:45,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:45,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:45,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878745141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878745141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:45,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:45,363 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/ab6759bb8fb347449d28e0d3cd585ac1 2024-11-17T21:24:45,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/999fe71bd900436d8392e09e1034020d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/999fe71bd900436d8392e09e1034020d 2024-11-17T21:24:45,377 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/999fe71bd900436d8392e09e1034020d, entries=150, sequenceid=158, filesize=30.4 K 2024-11-17T21:24:45,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/92bd1421082a42d8a121bc4fe3455617 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/92bd1421082a42d8a121bc4fe3455617 2024-11-17T21:24:45,391 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/92bd1421082a42d8a121bc4fe3455617, entries=150, sequenceid=158, filesize=11.9 K 2024-11-17T21:24:45,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/ab6759bb8fb347449d28e0d3cd585ac1 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/ab6759bb8fb347449d28e0d3cd585ac1 2024-11-17T21:24:45,401 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/ab6759bb8fb347449d28e0d3cd585ac1, entries=150, sequenceid=158, filesize=11.9 K 2024-11-17T21:24:45,402 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 4554a61bae93fe0dedbab5f4c1513837 in 1356ms, sequenceid=158, compaction requested=false 2024-11-17T21:24:45,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:45,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:45,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-17T21:24:45,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-17T21:24:45,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-17T21:24:45,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4390 sec 2024-11-17T21:24:45,413 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.4460 sec 2024-11-17T21:24:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:45,443 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:45,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:24:45,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a1e015e9728547419cbfe29583e92b74_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878684815/Put/seqid=0 2024-11-17T21:24:45,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878745479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:45,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878745482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:45,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878745483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:45,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878745485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:45,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742010_1186 (size=14794) 2024-11-17T21:24:45,498 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:45,503 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a1e015e9728547419cbfe29583e92b74_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a1e015e9728547419cbfe29583e92b74_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:45,505 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/337a41c960a548cd9d61ad89e3d05240, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:45,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/337a41c960a548cd9d61ad89e3d05240 is 175, key is test_row_0/A:col10/1731878684815/Put/seqid=0 2024-11-17T21:24:45,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742011_1187 (size=39749) 2024-11-17T21:24:45,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878745591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:45,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878745591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:45,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878745592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:45,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878745592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:45,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878745801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878745801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:45,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878745801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:45,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878745801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:45,927 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/337a41c960a548cd9d61ad89e3d05240 2024-11-17T21:24:45,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/75fdc8ac69ec4db5b86f9a1a50875ffa is 50, key is test_row_0/B:col10/1731878684815/Put/seqid=0 2024-11-17T21:24:45,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742012_1188 (size=12151) 2024-11-17T21:24:45,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/75fdc8ac69ec4db5b86f9a1a50875ffa 2024-11-17T21:24:45,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/0708382940a3471588f6884ea742c569 is 50, key is test_row_0/C:col10/1731878684815/Put/seqid=0 2024-11-17T21:24:45,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742013_1189 (size=12151) 2024-11-17T21:24:46,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878746111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878746111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:46,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878746111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:46,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878746111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:46,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/0708382940a3471588f6884ea742c569 2024-11-17T21:24:46,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/337a41c960a548cd9d61ad89e3d05240 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/337a41c960a548cd9d61ad89e3d05240 2024-11-17T21:24:46,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/337a41c960a548cd9d61ad89e3d05240, entries=200, sequenceid=172, filesize=38.8 K 2024-11-17T21:24:46,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/75fdc8ac69ec4db5b86f9a1a50875ffa as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/75fdc8ac69ec4db5b86f9a1a50875ffa 2024-11-17T21:24:46,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/75fdc8ac69ec4db5b86f9a1a50875ffa, entries=150, sequenceid=172, filesize=11.9 K 2024-11-17T21:24:46,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/0708382940a3471588f6884ea742c569 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/0708382940a3471588f6884ea742c569 2024-11-17T21:24:46,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/0708382940a3471588f6884ea742c569, entries=150, sequenceid=172, filesize=11.9 K 2024-11-17T21:24:46,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4554a61bae93fe0dedbab5f4c1513837 in 939ms, sequenceid=172, compaction requested=true 2024-11-17T21:24:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:46,382 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:46,382 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:46,383 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:46,383 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:46,383 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/B is initiating minor compaction (all files) 2024-11-17T21:24:46,383 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/A is initiating minor compaction (all files) 2024-11-17T21:24:46,383 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/B in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:46,383 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/A in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:46,384 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/75d3585afa3e42c3a8423a73d37d0536, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/999fe71bd900436d8392e09e1034020d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/337a41c960a548cd9d61ad89e3d05240] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=99.9 K 2024-11-17T21:24:46,384 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/0c05cf32726c4422a86ddeab34ae6717, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/92bd1421082a42d8a121bc4fe3455617, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/75fdc8ac69ec4db5b86f9a1a50875ffa] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=35.9 K 2024-11-17T21:24:46,384 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:46,384 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/75d3585afa3e42c3a8423a73d37d0536, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/999fe71bd900436d8392e09e1034020d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/337a41c960a548cd9d61ad89e3d05240] 2024-11-17T21:24:46,384 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c05cf32726c4422a86ddeab34ae6717, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731878682519 2024-11-17T21:24:46,384 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75d3585afa3e42c3a8423a73d37d0536, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731878682519 2024-11-17T21:24:46,384 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 999fe71bd900436d8392e09e1034020d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731878682616 2024-11-17T21:24:46,385 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 92bd1421082a42d8a121bc4fe3455617, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731878682616 2024-11-17T21:24:46,385 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 75fdc8ac69ec4db5b86f9a1a50875ffa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731878684813 2024-11-17T21:24:46,385 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 337a41c960a548cd9d61ad89e3d05240, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731878684812 2024-11-17T21:24:46,392 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:46,393 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#B#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:46,394 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/1ea6ee5250504a43a1da6faa82e0b303 is 50, key is test_row_0/B:col10/1731878684815/Put/seqid=0 2024-11-17T21:24:46,394 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117c75a52bee9304e6fa1444cd1b90bfeb9_4554a61bae93fe0dedbab5f4c1513837 store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:46,397 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117c75a52bee9304e6fa1444cd1b90bfeb9_4554a61bae93fe0dedbab5f4c1513837, store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:46,397 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c75a52bee9304e6fa1444cd1b90bfeb9_4554a61bae93fe0dedbab5f4c1513837 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:46,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742014_1190 (size=12561) 2024-11-17T21:24:46,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742015_1191 (size=4469) 2024-11-17T21:24:46,415 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#A#compaction#163 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:46,416 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/d9aeb9552ea04252bb98e480e6dc498a is 175, key is test_row_0/A:col10/1731878684815/Put/seqid=0 2024-11-17T21:24:46,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742016_1192 (size=31515) 2024-11-17T21:24:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:46,624 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:46,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T21:24:46,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878746631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878746633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878746631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878746633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:46,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111745a1e164c48a4fa2b7fa65ae8dce22a0_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878686623/Put/seqid=0 2024-11-17T21:24:46,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742017_1193 (size=14794) 2024-11-17T21:24:46,675 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:46,681 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111745a1e164c48a4fa2b7fa65ae8dce22a0_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111745a1e164c48a4fa2b7fa65ae8dce22a0_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:46,682 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/beb666c62c3b46f7b496949662526082, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:46,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/beb666c62c3b46f7b496949662526082 is 175, key is test_row_0/A:col10/1731878686623/Put/seqid=0 2024-11-17T21:24:46,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742018_1194 (size=39749) 2024-11-17T21:24:46,706 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/beb666c62c3b46f7b496949662526082 2024-11-17T21:24:46,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/8b59104c0c9243b6b408c279b23796cb is 50, key is test_row_0/B:col10/1731878686623/Put/seqid=0 2024-11-17T21:24:46,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742019_1195 (size=12151) 2024-11-17T21:24:46,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878746742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878746742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878746742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:46,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878746751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:46,815 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/1ea6ee5250504a43a1da6faa82e0b303 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/1ea6ee5250504a43a1da6faa82e0b303 2024-11-17T21:24:46,819 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/B of 4554a61bae93fe0dedbab5f4c1513837 into 1ea6ee5250504a43a1da6faa82e0b303(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:46,820 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:46,820 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/B, priority=13, startTime=1731878686382; duration=0sec 2024-11-17T21:24:46,820 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:46,820 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:B 2024-11-17T21:24:46,820 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:46,821 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:46,821 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/C is initiating minor compaction (all files) 2024-11-17T21:24:46,821 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/C in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:46,821 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/67e313311bdb46b7b5450c13d5d46181, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/ab6759bb8fb347449d28e0d3cd585ac1, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/0708382940a3471588f6884ea742c569] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=35.9 K 2024-11-17T21:24:46,821 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 67e313311bdb46b7b5450c13d5d46181, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731878682519 2024-11-17T21:24:46,822 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting ab6759bb8fb347449d28e0d3cd585ac1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731878682616 2024-11-17T21:24:46,822 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 0708382940a3471588f6884ea742c569, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731878684813 2024-11-17T21:24:46,829 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#C#compaction#166 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:46,829 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/54e25f5fb3a246219ca71bf9df72e2b4 is 50, key is test_row_0/C:col10/1731878684815/Put/seqid=0 2024-11-17T21:24:46,833 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/d9aeb9552ea04252bb98e480e6dc498a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/d9aeb9552ea04252bb98e480e6dc498a 2024-11-17T21:24:46,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878746834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,835 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_0' in 4554a61bae93fe0dedbab5f4c1513837 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:24:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:46,838 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/A of 4554a61bae93fe0dedbab5f4c1513837 into d9aeb9552ea04252bb98e480e6dc498a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:46,838 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:46,839 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/A, priority=13, startTime=1731878686382; duration=0sec 2024-11-17T21:24:46,839 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:46,839 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:A 2024-11-17T21:24:46,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742020_1196 (size=12561) 2024-11-17T21:24:46,850 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/54e25f5fb3a246219ca71bf9df72e2b4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/54e25f5fb3a246219ca71bf9df72e2b4 2024-11-17T21:24:46,854 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/C of 4554a61bae93fe0dedbab5f4c1513837 into 54e25f5fb3a246219ca71bf9df72e2b4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:46,854 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:46,854 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/C, priority=13, startTime=1731878686382; duration=0sec 2024-11-17T21:24:46,855 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:46,855 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:C 2024-11-17T21:24:46,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878746951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878746951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:46,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878746961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:46,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878746961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:47,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/8b59104c0c9243b6b408c279b23796cb 2024-11-17T21:24:47,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/1fc5d07c8d444c72aeab1172324f25ab is 50, key is test_row_0/C:col10/1731878686623/Put/seqid=0 2024-11-17T21:24:47,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742021_1197 (size=12151) 2024-11-17T21:24:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T21:24:47,141 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:47,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-17T21:24:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T21:24:47,144 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:47,145 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:47,145 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T21:24:47,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878747261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:47,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:47,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878747261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:47,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878747270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:47,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:47,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878747271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:47,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:47,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T21:24:47,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:47,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:47,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T21:24:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:47,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T21:24:47,498 INFO [master/a313eea8709e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-17T21:24:47,498 INFO [master/a313eea8709e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-17T21:24:47,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/1fc5d07c8d444c72aeab1172324f25ab 2024-11-17T21:24:47,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/beb666c62c3b46f7b496949662526082 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/beb666c62c3b46f7b496949662526082 2024-11-17T21:24:47,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/beb666c62c3b46f7b496949662526082, entries=200, sequenceid=198, filesize=38.8 K 2024-11-17T21:24:47,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/8b59104c0c9243b6b408c279b23796cb as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/8b59104c0c9243b6b408c279b23796cb 2024-11-17T21:24:47,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/8b59104c0c9243b6b408c279b23796cb, entries=150, sequenceid=198, filesize=11.9 K 2024-11-17T21:24:47,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/1fc5d07c8d444c72aeab1172324f25ab as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/1fc5d07c8d444c72aeab1172324f25ab 2024-11-17T21:24:47,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/1fc5d07c8d444c72aeab1172324f25ab, entries=150, sequenceid=198, filesize=11.9 K 2024-11-17T21:24:47,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 4554a61bae93fe0dedbab5f4c1513837 in 939ms, sequenceid=198, compaction requested=false 2024-11-17T21:24:47,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:47,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T21:24:47,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:47,604 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:24:47,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111777fb37d0cb474e4689ab472a21320b7f_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878686631/Put/seqid=0 2024-11-17T21:24:47,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742022_1198 (size=12304) 2024-11-17T21:24:47,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:47,635 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111777fb37d0cb474e4689ab472a21320b7f_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111777fb37d0cb474e4689ab472a21320b7f_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:47,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/425d274774014111b81c3eee05d2bf2f, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:47,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/425d274774014111b81c3eee05d2bf2f is 175, key is test_row_0/A:col10/1731878686631/Put/seqid=0 2024-11-17T21:24:47,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742023_1199 (size=31105) 2024-11-17T21:24:47,658 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/425d274774014111b81c3eee05d2bf2f 2024-11-17T21:24:47,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/ab88166947134fa7bbe0653b21a4fbbf is 50, key is test_row_0/B:col10/1731878686631/Put/seqid=0 2024-11-17T21:24:47,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742024_1200 (size=12151) 2024-11-17T21:24:47,679 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/ab88166947134fa7bbe0653b21a4fbbf 2024-11-17T21:24:47,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/8b92e8acccf942b9a06368caa0e25e84 is 50, key is test_row_0/C:col10/1731878686631/Put/seqid=0 2024-11-17T21:24:47,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742025_1201 (size=12151) 2024-11-17T21:24:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T21:24:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:47,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:47,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878747803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:47,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:47,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878747804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:47,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:47,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:47,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878747808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878747809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:47,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878747921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:47,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878747921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:47,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:47,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878747921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878747921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878748131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:48,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878748131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878748131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878748131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:48,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,165 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/8b92e8acccf942b9a06368caa0e25e84 2024-11-17T21:24:48,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/425d274774014111b81c3eee05d2bf2f as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/425d274774014111b81c3eee05d2bf2f 2024-11-17T21:24:48,173 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/425d274774014111b81c3eee05d2bf2f, entries=150, sequenceid=212, filesize=30.4 K 2024-11-17T21:24:48,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/ab88166947134fa7bbe0653b21a4fbbf as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/ab88166947134fa7bbe0653b21a4fbbf 2024-11-17T21:24:48,178 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/ab88166947134fa7bbe0653b21a4fbbf, entries=150, sequenceid=212, filesize=11.9 K 2024-11-17T21:24:48,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/8b92e8acccf942b9a06368caa0e25e84 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/8b92e8acccf942b9a06368caa0e25e84 2024-11-17T21:24:48,183 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/8b92e8acccf942b9a06368caa0e25e84, entries=150, sequenceid=212, filesize=11.9 K 2024-11-17T21:24:48,184 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4554a61bae93fe0dedbab5f4c1513837 in 580ms, sequenceid=212, compaction requested=true 2024-11-17T21:24:48,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:48,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:48,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-17T21:24:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-17T21:24:48,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-17T21:24:48,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0400 sec 2024-11-17T21:24:48,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.0460 sec 2024-11-17T21:24:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T21:24:48,281 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:48,281 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-17T21:24:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T21:24:48,283 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:48,284 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:48,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T21:24:48,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T21:24:48,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:48,438 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T21:24:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:48,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:48,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878748448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:48,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173d4d0c1f4be648bdbf813ce2283b9a86_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878687808/Put/seqid=0 2024-11-17T21:24:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878748450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:48,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878748451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878748453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742026_1202 (size=12304) 2024-11-17T21:24:48,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878748561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878748561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878748561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:48,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878748561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:48,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T21:24:48,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878748771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:48,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878748771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878748771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:48,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:48,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878748772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:48,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:48,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:48,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:48,870 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173d4d0c1f4be648bdbf813ce2283b9a86_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173d4d0c1f4be648bdbf813ce2283b9a86_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:48,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/e208d16c2b404d429693ab751e8e0a68, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:48,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/e208d16c2b404d429693ab751e8e0a68 is 175, key is test_row_0/A:col10/1731878687808/Put/seqid=0 2024-11-17T21:24:48,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742027_1203 (size=31105) 2024-11-17T21:24:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T21:24:49,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878749081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:49,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878749081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:49,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:49,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:49,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878749081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:49,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:49,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:49,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878749081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:49,276 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/e208d16c2b404d429693ab751e8e0a68 2024-11-17T21:24:49,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/d9747076379f442c86c8b316fcfe86b4 is 50, key is test_row_0/B:col10/1731878687808/Put/seqid=0 2024-11-17T21:24:49,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742028_1204 (size=12151) 2024-11-17T21:24:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T21:24:49,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878749591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:49,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878749591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:49,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:49,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:49,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878749591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:49,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878749592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:49,691 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/d9747076379f442c86c8b316fcfe86b4 2024-11-17T21:24:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/20c3a71bc2644fff928f02ddb2b38cc4 is 50, key is test_row_0/C:col10/1731878687808/Put/seqid=0 2024-11-17T21:24:49,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742029_1205 (size=12151) 2024-11-17T21:24:50,109 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/20c3a71bc2644fff928f02ddb2b38cc4 2024-11-17T21:24:50,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/e208d16c2b404d429693ab751e8e0a68 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/e208d16c2b404d429693ab751e8e0a68 2024-11-17T21:24:50,120 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/e208d16c2b404d429693ab751e8e0a68, entries=150, sequenceid=237, filesize=30.4 K 2024-11-17T21:24:50,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/d9747076379f442c86c8b316fcfe86b4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/d9747076379f442c86c8b316fcfe86b4 2024-11-17T21:24:50,125 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/d9747076379f442c86c8b316fcfe86b4, entries=150, sequenceid=237, filesize=11.9 K 2024-11-17T21:24:50,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/20c3a71bc2644fff928f02ddb2b38cc4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/20c3a71bc2644fff928f02ddb2b38cc4 2024-11-17T21:24:50,131 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/20c3a71bc2644fff928f02ddb2b38cc4, entries=150, sequenceid=237, filesize=11.9 K 2024-11-17T21:24:50,132 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4554a61bae93fe0dedbab5f4c1513837 in 1694ms, sequenceid=237, compaction requested=true 2024-11-17T21:24:50,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:50,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:50,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-17T21:24:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-17T21:24:50,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-17T21:24:50,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8500 sec 2024-11-17T21:24:50,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.8550 sec 2024-11-17T21:24:50,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T21:24:50,431 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:50,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:50,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-17T21:24:50,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T21:24:50,433 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:50,434 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:50,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:50,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T21:24:50,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T21:24:50,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:50,586 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:24:50,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175a3beada52614facb773ee8f7f323cd7_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878688448/Put/seqid=0 2024-11-17T21:24:50,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742030_1206 (size=12304) 2024-11-17T21:24:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:50,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:50,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878750632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:50,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:50,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878750634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:50,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:50,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878750635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:50,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878750636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:50,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:50,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878750741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878750741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878750741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T21:24:50,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878750751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:50,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38464 deadline: 1731878750881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:50,886 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_0' in 4554a61bae93fe0dedbab5f4c1513837 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:24:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:50,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878750951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878750952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878750952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:50,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:50,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878750961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:50,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:51,005 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175a3beada52614facb773ee8f7f323cd7_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175a3beada52614facb773ee8f7f323cd7_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:51,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/74a70577559a4ab4bc5be928d0a8475e, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:51,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/74a70577559a4ab4bc5be928d0a8475e is 175, key is test_row_0/A:col10/1731878688448/Put/seqid=0 2024-11-17T21:24:51,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742031_1207 (size=31105) 2024-11-17T21:24:51,018 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/74a70577559a4ab4bc5be928d0a8475e 2024-11-17T21:24:51,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/b6beded0c5404284b6dd2bfa7ec11f6f is 50, key is test_row_0/B:col10/1731878688448/Put/seqid=0 2024-11-17T21:24:51,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742032_1208 (size=12151) 2024-11-17T21:24:51,037 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/b6beded0c5404284b6dd2bfa7ec11f6f 2024-11-17T21:24:51,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/a65fa05a3a144aab9405380a52e8e7c4 is 50, key is test_row_0/C:col10/1731878688448/Put/seqid=0 2024-11-17T21:24:51,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742033_1209 (size=12151) 2024-11-17T21:24:51,059 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/a65fa05a3a144aab9405380a52e8e7c4 2024-11-17T21:24:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T21:24:51,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/74a70577559a4ab4bc5be928d0a8475e as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/74a70577559a4ab4bc5be928d0a8475e 2024-11-17T21:24:51,076 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/74a70577559a4ab4bc5be928d0a8475e, entries=150, sequenceid=248, filesize=30.4 K 2024-11-17T21:24:51,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/b6beded0c5404284b6dd2bfa7ec11f6f as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/b6beded0c5404284b6dd2bfa7ec11f6f 2024-11-17T21:24:51,086 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/b6beded0c5404284b6dd2bfa7ec11f6f, entries=150, sequenceid=248, filesize=11.9 K 2024-11-17T21:24:51,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/a65fa05a3a144aab9405380a52e8e7c4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/a65fa05a3a144aab9405380a52e8e7c4 2024-11-17T21:24:51,101 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/a65fa05a3a144aab9405380a52e8e7c4, entries=150, sequenceid=248, filesize=11.9 K 2024-11-17T21:24:51,102 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4554a61bae93fe0dedbab5f4c1513837 in 516ms, sequenceid=248, compaction requested=true 2024-11-17T21:24:51,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:51,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:51,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-17T21:24:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-17T21:24:51,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-17T21:24:51,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 670 msec 2024-11-17T21:24:51,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 676 msec 2024-11-17T21:24:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:51,263 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:51,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T21:24:51,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a06830281ce14e33a5d466ad6586bc01_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878690634/Put/seqid=0 2024-11-17T21:24:51,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:51,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878751273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878751273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:51,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:51,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878751276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878751275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:51,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742034_1210 (size=12454) 2024-11-17T21:24:51,300 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:51,307 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a06830281ce14e33a5d466ad6586bc01_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a06830281ce14e33a5d466ad6586bc01_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:51,308 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/ec0941b75d9748ce85c8a72506332fda, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:51,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/ec0941b75d9748ce85c8a72506332fda is 175, key is test_row_0/A:col10/1731878690634/Put/seqid=0 2024-11-17T21:24:51,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742035_1211 (size=31255) 2024-11-17T21:24:51,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878751391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:51,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:51,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878751391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:51,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:51,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:51,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878751391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:51,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:51,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878751392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:51,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T21:24:51,571 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:51,572 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-17T21:24:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T21:24:51,574 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:51,575 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:51,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:51,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878751601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:51,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878751601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:51,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878751601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878751601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T21:24:51,723 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/ec0941b75d9748ce85c8a72506332fda 2024-11-17T21:24:51,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T21:24:51,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:51,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:51,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:51,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:51,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:51,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/c84f899fff3546bb8f43c64a286cc934 is 50, key is test_row_0/B:col10/1731878690634/Put/seqid=0 2024-11-17T21:24:51,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742036_1212 (size=12301) 2024-11-17T21:24:51,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T21:24:51,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:51,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:51,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:51,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:51,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T21:24:51,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878751910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:51,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878751911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878751911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:51,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:51,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878751921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:52,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T21:24:52,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:52,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:52,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:52,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:52,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:52,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:52,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/c84f899fff3546bb8f43c64a286cc934 2024-11-17T21:24:52,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/30fbc808742141c3b54043d1eea92303 is 50, key is test_row_0/C:col10/1731878690634/Put/seqid=0 2024-11-17T21:24:52,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742037_1213 (size=12301) 2024-11-17T21:24:52,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T21:24:52,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:52,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:52,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:52,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:52,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T21:24:52,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T21:24:52,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:52,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:52,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:52,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:52,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:52,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878752421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:52,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:52,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:52,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:52,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878752421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:52,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:52,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:52,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:52,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878752421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:52,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:52,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:52,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878752430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:52,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:52,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:52,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:52,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T21:24:52,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:52,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:52,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:52,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:52,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:52,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/30fbc808742141c3b54043d1eea92303 2024-11-17T21:24:52,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/ec0941b75d9748ce85c8a72506332fda as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/ec0941b75d9748ce85c8a72506332fda 2024-11-17T21:24:52,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/ec0941b75d9748ce85c8a72506332fda, entries=150, sequenceid=274, filesize=30.5 K 2024-11-17T21:24:52,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/c84f899fff3546bb8f43c64a286cc934 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/c84f899fff3546bb8f43c64a286cc934 2024-11-17T21:24:52,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/c84f899fff3546bb8f43c64a286cc934, entries=150, sequenceid=274, filesize=12.0 K 2024-11-17T21:24:52,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/30fbc808742141c3b54043d1eea92303 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/30fbc808742141c3b54043d1eea92303 2024-11-17T21:24:52,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/30fbc808742141c3b54043d1eea92303, entries=150, sequenceid=274, filesize=12.0 K 2024-11-17T21:24:52,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 4554a61bae93fe0dedbab5f4c1513837 in 1308ms, sequenceid=274, compaction requested=true 2024-11-17T21:24:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:52,571 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-17T21:24:52,571 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-17T21:24:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:52,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:52,573 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73466 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-17T21:24:52,573 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/B is initiating minor compaction (all files) 2024-11-17T21:24:52,573 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/B in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:52,573 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/1ea6ee5250504a43a1da6faa82e0b303, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/8b59104c0c9243b6b408c279b23796cb, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/ab88166947134fa7bbe0653b21a4fbbf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/d9747076379f442c86c8b316fcfe86b4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/b6beded0c5404284b6dd2bfa7ec11f6f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/c84f899fff3546bb8f43c64a286cc934] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=71.7 K 2024-11-17T21:24:52,573 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 195834 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-17T21:24:52,573 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/A is initiating minor compaction (all files) 2024-11-17T21:24:52,573 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/A in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:52,573 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/d9aeb9552ea04252bb98e480e6dc498a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/beb666c62c3b46f7b496949662526082, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/425d274774014111b81c3eee05d2bf2f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/e208d16c2b404d429693ab751e8e0a68, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/74a70577559a4ab4bc5be928d0a8475e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/ec0941b75d9748ce85c8a72506332fda] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=191.2 K 2024-11-17T21:24:52,573 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:52,573 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/d9aeb9552ea04252bb98e480e6dc498a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/beb666c62c3b46f7b496949662526082, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/425d274774014111b81c3eee05d2bf2f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/e208d16c2b404d429693ab751e8e0a68, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/74a70577559a4ab4bc5be928d0a8475e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/ec0941b75d9748ce85c8a72506332fda] 2024-11-17T21:24:52,574 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9aeb9552ea04252bb98e480e6dc498a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731878684813 2024-11-17T21:24:52,574 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ea6ee5250504a43a1da6faa82e0b303, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731878684813 2024-11-17T21:24:52,574 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting beb666c62c3b46f7b496949662526082, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731878685469 2024-11-17T21:24:52,574 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b59104c0c9243b6b408c279b23796cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731878685481 2024-11-17T21:24:52,574 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 425d274774014111b81c3eee05d2bf2f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731878686630 2024-11-17T21:24:52,574 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting ab88166947134fa7bbe0653b21a4fbbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731878686630 2024-11-17T21:24:52,574 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting e208d16c2b404d429693ab751e8e0a68, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731878687802 2024-11-17T21:24:52,575 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d9747076379f442c86c8b316fcfe86b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731878687802 2024-11-17T21:24:52,575 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74a70577559a4ab4bc5be928d0a8475e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731878688444 2024-11-17T21:24:52,575 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting b6beded0c5404284b6dd2bfa7ec11f6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731878688444 2024-11-17T21:24:52,575 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec0941b75d9748ce85c8a72506332fda, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731878690633 2024-11-17T21:24:52,575 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting c84f899fff3546bb8f43c64a286cc934, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731878690633 2024-11-17T21:24:52,584 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:52,587 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111726904f963419471588312580c6e30f73_4554a61bae93fe0dedbab5f4c1513837 store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:52,589 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#B#compaction#181 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:52,590 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/3cad7050f2624c69a6227d876b7f2bfd is 50, key is test_row_0/B:col10/1731878690634/Put/seqid=0 2024-11-17T21:24:52,590 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111726904f963419471588312580c6e30f73_4554a61bae93fe0dedbab5f4c1513837, store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:52,590 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111726904f963419471588312580c6e30f73_4554a61bae93fe0dedbab5f4c1513837 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:52,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742038_1214 (size=12915) 2024-11-17T21:24:52,606 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/3cad7050f2624c69a6227d876b7f2bfd as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3cad7050f2624c69a6227d876b7f2bfd 2024-11-17T21:24:52,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742039_1215 (size=4469) 2024-11-17T21:24:52,613 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/B of 4554a61bae93fe0dedbab5f4c1513837 into 3cad7050f2624c69a6227d876b7f2bfd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:52,613 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:52,613 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/B, priority=10, startTime=1731878692571; duration=0sec 2024-11-17T21:24:52,613 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:52,613 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:B 2024-11-17T21:24:52,613 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-17T21:24:52,615 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73466 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-17T21:24:52,615 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/C is initiating minor compaction (all files) 2024-11-17T21:24:52,615 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/C in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:52,615 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/54e25f5fb3a246219ca71bf9df72e2b4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/1fc5d07c8d444c72aeab1172324f25ab, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/8b92e8acccf942b9a06368caa0e25e84, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/20c3a71bc2644fff928f02ddb2b38cc4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/a65fa05a3a144aab9405380a52e8e7c4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/30fbc808742141c3b54043d1eea92303] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=71.7 K 2024-11-17T21:24:52,615 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 54e25f5fb3a246219ca71bf9df72e2b4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731878684813 2024-11-17T21:24:52,617 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fc5d07c8d444c72aeab1172324f25ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731878685481 2024-11-17T21:24:52,617 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b92e8acccf942b9a06368caa0e25e84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731878686630 2024-11-17T21:24:52,617 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 20c3a71bc2644fff928f02ddb2b38cc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731878687802 2024-11-17T21:24:52,618 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting a65fa05a3a144aab9405380a52e8e7c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731878688444 2024-11-17T21:24:52,618 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 30fbc808742141c3b54043d1eea92303, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731878690633 2024-11-17T21:24:52,629 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#C#compaction#182 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:52,629 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/bf3aa277a26f4919b11cbeb504eca5a9 is 50, key is test_row_0/C:col10/1731878690634/Put/seqid=0 2024-11-17T21:24:52,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742040_1216 (size=12915) 2024-11-17T21:24:52,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T21:24:52,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:52,646 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/bf3aa277a26f4919b11cbeb504eca5a9 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/bf3aa277a26f4919b11cbeb504eca5a9 2024-11-17T21:24:52,646 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-17T21:24:52,654 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/C of 4554a61bae93fe0dedbab5f4c1513837 into bf3aa277a26f4919b11cbeb504eca5a9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:52,654 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:52,654 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/C, priority=10, startTime=1731878692571; duration=0sec 2024-11-17T21:24:52,654 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:52,654 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:C 2024-11-17T21:24:52,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a614c36e595a4d9c9954fddce203029d_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878691271/Put/seqid=0 2024-11-17T21:24:52,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742041_1217 (size=12454) 2024-11-17T21:24:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T21:24:53,013 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#A#compaction#180 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:53,014 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/47b0e57ed95d424d8e7f0b399199c678 is 175, key is test_row_0/A:col10/1731878690634/Put/seqid=0 2024-11-17T21:24:53,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742042_1218 (size=31869) 2024-11-17T21:24:53,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:53,070 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a614c36e595a4d9c9954fddce203029d_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a614c36e595a4d9c9954fddce203029d_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:53,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/36c716e10dc14ddbba050b630256707d, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:53,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/36c716e10dc14ddbba050b630256707d is 175, key is test_row_0/A:col10/1731878691271/Put/seqid=0 2024-11-17T21:24:53,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742043_1219 (size=31255) 2024-11-17T21:24:53,426 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/47b0e57ed95d424d8e7f0b399199c678 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/47b0e57ed95d424d8e7f0b399199c678 2024-11-17T21:24:53,433 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/A of 4554a61bae93fe0dedbab5f4c1513837 into 47b0e57ed95d424d8e7f0b399199c678(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:53,433 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:53,433 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/A, priority=10, startTime=1731878692571; duration=0sec 2024-11-17T21:24:53,433 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:53,433 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:A 2024-11-17T21:24:53,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:53,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:53,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878753465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:53,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878753465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:53,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:53,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878753468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:53,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:53,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878753471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:53,476 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/36c716e10dc14ddbba050b630256707d 2024-11-17T21:24:53,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/05a2fb65886d4b3dbba9fa6730466eae is 50, key is test_row_0/B:col10/1731878691271/Put/seqid=0 2024-11-17T21:24:53,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742044_1220 (size=12301) 2024-11-17T21:24:53,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878753581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:53,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878753581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:53,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878753581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878753581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T21:24:53,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878753791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:53,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878753791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:53,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:53,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878753791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878753791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:53,903 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/05a2fb65886d4b3dbba9fa6730466eae 2024-11-17T21:24:53,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/438299e1fbba4b38b00bedbb56708843 is 50, key is test_row_0/C:col10/1731878691271/Put/seqid=0 2024-11-17T21:24:53,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742045_1221 (size=12301) 2024-11-17T21:24:54,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878754101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:54,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:54,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:54,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878754101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878754101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:54,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878754101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:54,314 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/438299e1fbba4b38b00bedbb56708843 2024-11-17T21:24:54,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/36c716e10dc14ddbba050b630256707d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/36c716e10dc14ddbba050b630256707d 2024-11-17T21:24:54,326 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/36c716e10dc14ddbba050b630256707d, entries=150, sequenceid=285, filesize=30.5 K 2024-11-17T21:24:54,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/05a2fb65886d4b3dbba9fa6730466eae as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/05a2fb65886d4b3dbba9fa6730466eae 2024-11-17T21:24:54,332 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/05a2fb65886d4b3dbba9fa6730466eae, entries=150, sequenceid=285, filesize=12.0 K 2024-11-17T21:24:54,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/438299e1fbba4b38b00bedbb56708843 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/438299e1fbba4b38b00bedbb56708843 2024-11-17T21:24:54,341 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/438299e1fbba4b38b00bedbb56708843, entries=150, sequenceid=285, filesize=12.0 K 2024-11-17T21:24:54,342 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 4554a61bae93fe0dedbab5f4c1513837 in 1696ms, sequenceid=285, compaction requested=false 2024-11-17T21:24:54,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:54,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:54,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-17T21:24:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-17T21:24:54,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-17T21:24:54,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7690 sec 2024-11-17T21:24:54,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.7740 sec 2024-11-17T21:24:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:54,614 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:54,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-17T21:24:54,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117607bf69b2d32490fbb69186327de0b28_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878694613/Put/seqid=0 2024-11-17T21:24:54,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878754619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:54,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878754622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:54,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878754628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:54,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878754629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:54,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742046_1222 (size=14994) 2024-11-17T21:24:54,635 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:54,642 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117607bf69b2d32490fbb69186327de0b28_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117607bf69b2d32490fbb69186327de0b28_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:54,646 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/39897bb69837463fa7090803749593ea, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:54,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/39897bb69837463fa7090803749593ea is 175, key is test_row_0/A:col10/1731878694613/Put/seqid=0 2024-11-17T21:24:54,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742047_1223 (size=39949) 2024-11-17T21:24:54,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878754731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:54,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878754731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:54,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878754731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:54,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878754740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:54,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878754941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:54,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878754941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:54,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878754941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:54,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878754950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:55,056 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/39897bb69837463fa7090803749593ea 2024-11-17T21:24:55,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/11ba5059c1f7401f8bf1e16fcab2de83 is 50, key is test_row_0/B:col10/1731878694613/Put/seqid=0 2024-11-17T21:24:55,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742048_1224 (size=12301) 2024-11-17T21:24:55,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878755251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:55,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878755251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:55,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878755251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:55,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878755261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:55,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/11ba5059c1f7401f8bf1e16fcab2de83 2024-11-17T21:24:55,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/50c49b5b62004cd783933b24de22e646 is 50, key is test_row_0/C:col10/1731878694613/Put/seqid=0 2024-11-17T21:24:55,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742049_1225 (size=12301) 2024-11-17T21:24:55,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/50c49b5b62004cd783933b24de22e646 2024-11-17T21:24:55,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/39897bb69837463fa7090803749593ea as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/39897bb69837463fa7090803749593ea 2024-11-17T21:24:55,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/39897bb69837463fa7090803749593ea, entries=200, sequenceid=314, filesize=39.0 K 2024-11-17T21:24:55,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/11ba5059c1f7401f8bf1e16fcab2de83 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/11ba5059c1f7401f8bf1e16fcab2de83 2024-11-17T21:24:55,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/11ba5059c1f7401f8bf1e16fcab2de83, entries=150, sequenceid=314, filesize=12.0 K 2024-11-17T21:24:55,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/50c49b5b62004cd783933b24de22e646 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/50c49b5b62004cd783933b24de22e646 2024-11-17T21:24:55,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/50c49b5b62004cd783933b24de22e646, entries=150, sequenceid=314, filesize=12.0 K 2024-11-17T21:24:55,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 4554a61bae93fe0dedbab5f4c1513837 in 883ms, sequenceid=314, compaction requested=true 2024-11-17T21:24:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:24:55,497 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:24:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:55,497 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:24:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:55,498 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:55,498 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:55,498 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/B is initiating minor compaction (all files) 2024-11-17T21:24:55,498 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/A is initiating minor compaction (all files) 2024-11-17T21:24:55,498 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/A in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:55,498 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/B in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:55,498 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3cad7050f2624c69a6227d876b7f2bfd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/05a2fb65886d4b3dbba9fa6730466eae, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/11ba5059c1f7401f8bf1e16fcab2de83] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=36.6 K 2024-11-17T21:24:55,498 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/47b0e57ed95d424d8e7f0b399199c678, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/36c716e10dc14ddbba050b630256707d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/39897bb69837463fa7090803749593ea] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=100.7 K 2024-11-17T21:24:55,498 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:55,498 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/47b0e57ed95d424d8e7f0b399199c678, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/36c716e10dc14ddbba050b630256707d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/39897bb69837463fa7090803749593ea] 2024-11-17T21:24:55,499 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cad7050f2624c69a6227d876b7f2bfd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731878690633 2024-11-17T21:24:55,499 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47b0e57ed95d424d8e7f0b399199c678, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731878690633 2024-11-17T21:24:55,499 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36c716e10dc14ddbba050b630256707d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731878691271 2024-11-17T21:24:55,499 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 05a2fb65886d4b3dbba9fa6730466eae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731878691271 2024-11-17T21:24:55,500 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 11ba5059c1f7401f8bf1e16fcab2de83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731878693462 2024-11-17T21:24:55,500 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39897bb69837463fa7090803749593ea, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731878693462 2024-11-17T21:24:55,505 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:55,506 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#B#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:55,506 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/568255adc6dc4a62b15de49067adb8e8 is 50, key is test_row_0/B:col10/1731878694613/Put/seqid=0 2024-11-17T21:24:55,517 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117a77f43819d714569a29a77ebee7a5b77_4554a61bae93fe0dedbab5f4c1513837 store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:55,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742050_1226 (size=13017) 2024-11-17T21:24:55,518 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117a77f43819d714569a29a77ebee7a5b77_4554a61bae93fe0dedbab5f4c1513837, store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:55,518 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a77f43819d714569a29a77ebee7a5b77_4554a61bae93fe0dedbab5f4c1513837 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:55,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742051_1227 (size=4469) 2024-11-17T21:24:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T21:24:55,741 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:55,741 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:24:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-17T21:24:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T21:24:55,743 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:24:55,743 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:24:55,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:24:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:55,763 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:55,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:24:55,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117559fd49f92e44ba0b928a054d4e5a9cb_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878695762/Put/seqid=0 2024-11-17T21:24:55,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742052_1228 (size=17534) 2024-11-17T21:24:55,776 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:55,779 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117559fd49f92e44ba0b928a054d4e5a9cb_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117559fd49f92e44ba0b928a054d4e5a9cb_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:55,780 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/c99aecf97f0b4190bf98f2e48682bd61, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:55,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/c99aecf97f0b4190bf98f2e48682bd61 is 175, key is test_row_0/A:col10/1731878695762/Put/seqid=0 2024-11-17T21:24:55,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742053_1229 (size=48639) 2024-11-17T21:24:55,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878755793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:55,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:55,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878755793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:55,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:55,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:55,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878755794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:55,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:55,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878755794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:55,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T21:24:55,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T21:24:55,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:55,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:55,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:55,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:55,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:55,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878755901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:55,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878755901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878755901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:55,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878755911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:55,923 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/568255adc6dc4a62b15de49067adb8e8 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/568255adc6dc4a62b15de49067adb8e8 2024-11-17T21:24:55,927 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/B of 4554a61bae93fe0dedbab5f4c1513837 into 568255adc6dc4a62b15de49067adb8e8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:55,927 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:55,927 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/B, priority=13, startTime=1731878695497; duration=0sec 2024-11-17T21:24:55,927 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:24:55,927 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:B 2024-11-17T21:24:55,927 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:24:55,928 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:24:55,928 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/C is initiating minor compaction (all files) 2024-11-17T21:24:55,928 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/C in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:55,928 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/bf3aa277a26f4919b11cbeb504eca5a9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/438299e1fbba4b38b00bedbb56708843, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/50c49b5b62004cd783933b24de22e646] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=36.6 K 2024-11-17T21:24:55,929 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting bf3aa277a26f4919b11cbeb504eca5a9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731878690633 2024-11-17T21:24:55,929 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 438299e1fbba4b38b00bedbb56708843, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731878691271 2024-11-17T21:24:55,930 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 50c49b5b62004cd783933b24de22e646, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731878693462 2024-11-17T21:24:55,936 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#C#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:55,936 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/a31bcf78134a4c81bfac85ef11cd29a9 is 50, key is test_row_0/C:col10/1731878694613/Put/seqid=0 2024-11-17T21:24:55,939 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#A#compaction#190 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:24:55,939 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/3cb53fd3eab74b8daf3c0d14759e5db7 is 175, key is test_row_0/A:col10/1731878694613/Put/seqid=0 2024-11-17T21:24:55,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742054_1230 (size=13017) 2024-11-17T21:24:55,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742055_1231 (size=31971) 2024-11-17T21:24:56,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T21:24:56,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:56,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:56,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:56,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:56,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T21:24:56,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:56,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878756111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878756111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:56,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878756111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:56,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878756121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:56,185 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/c99aecf97f0b4190bf98f2e48682bd61 2024-11-17T21:24:56,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/d05a6ac9d22a4bc09261952afc52ceb4 is 50, key is test_row_0/B:col10/1731878695762/Put/seqid=0 2024-11-17T21:24:56,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T21:24:56,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:56,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:56,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:56,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:56,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:56,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742056_1232 (size=12301) 2024-11-17T21:24:56,346 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/a31bcf78134a4c81bfac85ef11cd29a9 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/a31bcf78134a4c81bfac85ef11cd29a9 2024-11-17T21:24:56,349 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/3cb53fd3eab74b8daf3c0d14759e5db7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3cb53fd3eab74b8daf3c0d14759e5db7 2024-11-17T21:24:56,351 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/C of 4554a61bae93fe0dedbab5f4c1513837 into a31bcf78134a4c81bfac85ef11cd29a9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:56,352 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:56,352 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/C, priority=13, startTime=1731878695497; duration=0sec 2024-11-17T21:24:56,352 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:56,352 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:C 2024-11-17T21:24:56,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T21:24:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:56,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:56,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:56,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:56,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:56,356 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/A of 4554a61bae93fe0dedbab5f4c1513837 into 3cb53fd3eab74b8daf3c0d14759e5db7(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:24:56,356 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:56,356 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/A, priority=13, startTime=1731878695497; duration=0sec 2024-11-17T21:24:56,356 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:24:56,356 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:A 2024-11-17T21:24:56,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T21:24:56,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878756420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:56,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878756420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878756421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:56,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878756431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:56,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T21:24:56,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:56,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:56,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:56,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:56,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:24:56,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/d05a6ac9d22a4bc09261952afc52ceb4 2024-11-17T21:24:56,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/310d8113d07d4b85b3f5f9e4207157ff is 50, key is test_row_0/C:col10/1731878695762/Put/seqid=0 2024-11-17T21:24:56,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742057_1233 (size=12301) 2024-11-17T21:24:56,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/310d8113d07d4b85b3f5f9e4207157ff 2024-11-17T21:24:56,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/c99aecf97f0b4190bf98f2e48682bd61 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/c99aecf97f0b4190bf98f2e48682bd61 2024-11-17T21:24:56,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/c99aecf97f0b4190bf98f2e48682bd61, entries=250, sequenceid=326, filesize=47.5 K 2024-11-17T21:24:56,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/d05a6ac9d22a4bc09261952afc52ceb4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/d05a6ac9d22a4bc09261952afc52ceb4 2024-11-17T21:24:56,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/d05a6ac9d22a4bc09261952afc52ceb4, entries=150, sequenceid=326, filesize=12.0 K 2024-11-17T21:24:56,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/310d8113d07d4b85b3f5f9e4207157ff as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/310d8113d07d4b85b3f5f9e4207157ff 2024-11-17T21:24:56,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/310d8113d07d4b85b3f5f9e4207157ff, entries=150, sequenceid=326, filesize=12.0 K 2024-11-17T21:24:56,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 4554a61bae93fe0dedbab5f4c1513837 in 893ms, sequenceid=326, compaction requested=false 2024-11-17T21:24:56,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:56,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T21:24:56,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:56,660 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T21:24:56,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b4dc34af043f412dbd3f8295fd336f9e_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878695793/Put/seqid=0 2024-11-17T21:24:56,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742058_1234 (size=12454) 2024-11-17T21:24:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T21:24:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:56,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. as already flushing 2024-11-17T21:24:56,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878756937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:56,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878756941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:56,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:56,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878756941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878756941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:56,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:56,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:56,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:57,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878757051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:57,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878757051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:57,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878757051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:57,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:57,074 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b4dc34af043f412dbd3f8295fd336f9e_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b4dc34af043f412dbd3f8295fd336f9e_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:57,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/3edad44b93744ba19d5a6fb7551a7b81, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:57,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/3edad44b93744ba19d5a6fb7551a7b81 is 175, key is test_row_0/A:col10/1731878695793/Put/seqid=0 2024-11-17T21:24:57,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742059_1235 (size=31255) 2024-11-17T21:24:57,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878757261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:57,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:57,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878757261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:57,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:57,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878757261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:57,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:57,479 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/3edad44b93744ba19d5a6fb7551a7b81 2024-11-17T21:24:57,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/c6d33eb8a5b04e829352d15019d7fb0f is 50, key is test_row_0/B:col10/1731878695793/Put/seqid=0 2024-11-17T21:24:57,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742060_1236 (size=12301) 2024-11-17T21:24:57,534 INFO [Thread-707 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-707. 2024-11-17T21:24:57,534 DEBUG [Thread-707 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:57,534 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:57,534 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:57,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:57,536 INFO [Thread-709 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-709. 2024-11-17T21:24:57,536 DEBUG [Thread-709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:57,536 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:57,536 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:57,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:57,541 INFO [Thread-711 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-711. 2024-11-17T21:24:57,541 DEBUG [Thread-711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:57,541 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:57,541 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:57,541 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:57,543 INFO [Thread-713 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-713. 2024-11-17T21:24:57,543 DEBUG [Thread-713 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:57,543 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:57,543 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:57,543 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:57,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:57,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:57,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878757570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878757571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878757570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:57,891 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/c6d33eb8a5b04e829352d15019d7fb0f 2024-11-17T21:24:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T21:24:57,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/4f7f5a185ddd419caa978b8250e88405 is 50, key is test_row_0/C:col10/1731878695793/Put/seqid=0 2024-11-17T21:24:57,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742061_1237 (size=12301) 2024-11-17T21:24:57,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:57,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38450 deadline: 1731878757961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:57,962 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_0' in 4554a61bae93fe0dedbab5f4c1513837 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:24:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:58,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:58,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1731878758081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:58,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:24:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38462 deadline: 1731878758081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1731878758081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 2024-11-17T21:24:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:24:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:24:58,353 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/4f7f5a185ddd419caa978b8250e88405 2024-11-17T21:24:58,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/3edad44b93744ba19d5a6fb7551a7b81 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3edad44b93744ba19d5a6fb7551a7b81 2024-11-17T21:24:58,368 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3edad44b93744ba19d5a6fb7551a7b81, entries=150, sequenceid=354, filesize=30.5 K 2024-11-17T21:24:58,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/c6d33eb8a5b04e829352d15019d7fb0f as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/c6d33eb8a5b04e829352d15019d7fb0f 2024-11-17T21:24:58,373 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/c6d33eb8a5b04e829352d15019d7fb0f, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T21:24:58,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/4f7f5a185ddd419caa978b8250e88405 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/4f7f5a185ddd419caa978b8250e88405 2024-11-17T21:24:58,379 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/4f7f5a185ddd419caa978b8250e88405, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T21:24:58,379 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4554a61bae93fe0dedbab5f4c1513837 in 1719ms, sequenceid=354, compaction requested=true 2024-11-17T21:24:58,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:24:58,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:24:58,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-17T21:24:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-17T21:24:58,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-17T21:24:58,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6380 sec 2024-11-17T21:24:58,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.6420 sec 2024-11-17T21:24:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:59,096 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:24:59,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T21:24:59,097 INFO [Thread-698 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-698. 2024-11-17T21:24:59,097 INFO [Thread-704 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-704. 2024-11-17T21:24:59,097 INFO [Thread-696 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-696. 2024-11-17T21:24:59,097 DEBUG [Thread-698 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:59,097 DEBUG [Thread-696 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:59,097 DEBUG [Thread-704 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:59,097 DEBUG [Thread-698 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:59,097 DEBUG [Thread-696 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:59,097 DEBUG [Thread-704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:59,097 DEBUG [Thread-696 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:59,098 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:59,098 DEBUG [Thread-698 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:59,098 DEBUG [Thread-704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:59,098 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:59,098 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:24:59,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c6ad99929bb34d478a8533381c8cef12_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878696939/Put/seqid=0 2024-11-17T21:24:59,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742062_1238 (size=12454) 2024-11-17T21:24:59,510 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:24:59,514 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c6ad99929bb34d478a8533381c8cef12_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c6ad99929bb34d478a8533381c8cef12_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:24:59,514 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/9a6bd86963e04fcc89c3e2d2dbeeb9c2, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:24:59,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/9a6bd86963e04fcc89c3e2d2dbeeb9c2 is 175, key is test_row_0/A:col10/1731878696939/Put/seqid=0 2024-11-17T21:24:59,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742063_1239 (size=31255) 2024-11-17T21:24:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T21:24:59,911 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:24:59,919 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/9a6bd86963e04fcc89c3e2d2dbeeb9c2 2024-11-17T21:24:59,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/e6f4359c56c246f2b88af9fa4d07582f is 50, key is test_row_0/B:col10/1731878696939/Put/seqid=0 2024-11-17T21:24:59,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742064_1240 (size=12301) 2024-11-17T21:24:59,973 INFO [Thread-702 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-702. 2024-11-17T21:24:59,973 DEBUG [Thread-702 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:24:59,973 DEBUG [Thread-702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:59,973 DEBUG [Thread-702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:24:59,973 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:00,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/e6f4359c56c246f2b88af9fa4d07582f 2024-11-17T21:25:00,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/66d48c45ba964c0d8d873da4ccbbb8a9 is 50, key is test_row_0/C:col10/1731878696939/Put/seqid=0 2024-11-17T21:25:00,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742065_1241 (size=12301) 2024-11-17T21:25:00,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/66d48c45ba964c0d8d873da4ccbbb8a9 2024-11-17T21:25:00,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/9a6bd86963e04fcc89c3e2d2dbeeb9c2 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9a6bd86963e04fcc89c3e2d2dbeeb9c2 2024-11-17T21:25:00,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9a6bd86963e04fcc89c3e2d2dbeeb9c2, entries=150, sequenceid=368, filesize=30.5 K 2024-11-17T21:25:00,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/e6f4359c56c246f2b88af9fa4d07582f as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/e6f4359c56c246f2b88af9fa4d07582f 2024-11-17T21:25:00,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/e6f4359c56c246f2b88af9fa4d07582f, entries=150, sequenceid=368, filesize=12.0 K 2024-11-17T21:25:00,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/66d48c45ba964c0d8d873da4ccbbb8a9 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/66d48c45ba964c0d8d873da4ccbbb8a9 2024-11-17T21:25:00,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/66d48c45ba964c0d8d873da4ccbbb8a9, entries=150, sequenceid=368, filesize=12.0 K 2024-11-17T21:25:00,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=6.71 KB/6870 for 4554a61bae93fe0dedbab5f4c1513837 in 1701ms, sequenceid=368, compaction requested=true 2024-11-17T21:25:00,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:25:00,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:25:00,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:00,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:00,797 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:00,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:00,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4554a61bae93fe0dedbab5f4c1513837:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:25:00,797 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:00,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:00,799 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:00,799 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143120 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:00,799 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/A is initiating minor compaction (all files) 2024-11-17T21:25:00,799 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 4554a61bae93fe0dedbab5f4c1513837/B is initiating minor compaction (all files) 2024-11-17T21:25:00,799 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/B in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:25:00,799 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4554a61bae93fe0dedbab5f4c1513837/A in TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:25:00,799 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3cb53fd3eab74b8daf3c0d14759e5db7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/c99aecf97f0b4190bf98f2e48682bd61, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3edad44b93744ba19d5a6fb7551a7b81, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9a6bd86963e04fcc89c3e2d2dbeeb9c2] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=139.8 K 2024-11-17T21:25:00,799 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/568255adc6dc4a62b15de49067adb8e8, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/d05a6ac9d22a4bc09261952afc52ceb4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/c6d33eb8a5b04e829352d15019d7fb0f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/e6f4359c56c246f2b88af9fa4d07582f] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp, totalSize=48.8 K 2024-11-17T21:25:00,799 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:25:00,799 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3cb53fd3eab74b8daf3c0d14759e5db7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/c99aecf97f0b4190bf98f2e48682bd61, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3edad44b93744ba19d5a6fb7551a7b81, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9a6bd86963e04fcc89c3e2d2dbeeb9c2] 2024-11-17T21:25:00,800 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 568255adc6dc4a62b15de49067adb8e8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731878693462 2024-11-17T21:25:00,800 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cb53fd3eab74b8daf3c0d14759e5db7, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731878693462 2024-11-17T21:25:00,800 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d05a6ac9d22a4bc09261952afc52ceb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731878694616 2024-11-17T21:25:00,800 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting c99aecf97f0b4190bf98f2e48682bd61, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731878694616 2024-11-17T21:25:00,800 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3edad44b93744ba19d5a6fb7551a7b81, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731878695792 2024-11-17T21:25:00,800 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting c6d33eb8a5b04e829352d15019d7fb0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731878695792 2024-11-17T21:25:00,801 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a6bd86963e04fcc89c3e2d2dbeeb9c2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731878696939 2024-11-17T21:25:00,801 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting e6f4359c56c246f2b88af9fa4d07582f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731878696939 2024-11-17T21:25:00,812 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:25:00,815 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411173b546bb8d57f46f88ec68e786a93c52a_4554a61bae93fe0dedbab5f4c1513837 store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:25:00,815 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#B#compaction#201 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:00,815 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/a54200aaddba4d92bf9ea750aaa7922c is 50, key is test_row_0/B:col10/1731878696939/Put/seqid=0 2024-11-17T21:25:00,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742066_1242 (size=13153) 2024-11-17T21:25:00,853 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411173b546bb8d57f46f88ec68e786a93c52a_4554a61bae93fe0dedbab5f4c1513837, store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:25:00,853 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173b546bb8d57f46f88ec68e786a93c52a_4554a61bae93fe0dedbab5f4c1513837 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:25:00,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742067_1243 (size=4469) 2024-11-17T21:25:00,994 INFO [Thread-700 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-700. 2024-11-17T21:25:00,994 DEBUG [Thread-700 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:00,994 DEBUG [Thread-700 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:00,995 DEBUG [Thread-700 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:00,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:00,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T21:25:00,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-17T21:25:00,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-17T21:25:00,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-17T21:25:00,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-17T21:25:00,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-17T21:25:00,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T21:25:00,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3805 2024-11-17T21:25:00,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3623 2024-11-17T21:25:00,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T21:25:00,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1619 2024-11-17T21:25:00,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4855 rows 2024-11-17T21:25:00,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1618 2024-11-17T21:25:00,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4851 rows 2024-11-17T21:25:00,995 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T21:25:00,995 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T21:25:00,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:00,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:00,996 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:00,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T21:25:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T21:25:01,000 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878700999"}]},"ts":"1731878700999"} 2024-11-17T21:25:01,001 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T21:25:01,001 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T21:25:01,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T21:25:01,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4554a61bae93fe0dedbab5f4c1513837, UNASSIGN}] 2024-11-17T21:25:01,004 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4554a61bae93fe0dedbab5f4c1513837, UNASSIGN 2024-11-17T21:25:01,005 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=4554a61bae93fe0dedbab5f4c1513837, regionState=CLOSING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:25:01,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4554a61bae93fe0dedbab5f4c1513837, UNASSIGN because future has completed 2024-11-17T21:25:01,006 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T21:25:01,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:25:01,089 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T21:25:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T21:25:01,159 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:01,159 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T21:25:01,160 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing 4554a61bae93fe0dedbab5f4c1513837, disabling compactions & flushes 2024-11-17T21:25:01,160 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:25:01,234 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/a54200aaddba4d92bf9ea750aaa7922c as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/a54200aaddba4d92bf9ea750aaa7922c 2024-11-17T21:25:01,241 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/B of 4554a61bae93fe0dedbab5f4c1513837 into a54200aaddba4d92bf9ea750aaa7922c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:01,241 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:25:01,241 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/B, priority=12, startTime=1731878700797; duration=0sec 2024-11-17T21:25:01,241 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:01,241 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:B 2024-11-17T21:25:01,241 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. because compaction request was cancelled 2024-11-17T21:25:01,241 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:C 2024-11-17T21:25:01,264 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4554a61bae93fe0dedbab5f4c1513837#A#compaction#202 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:01,264 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/b4005db4faae4cfba0badb0376784194 is 175, key is test_row_0/A:col10/1731878696939/Put/seqid=0 2024-11-17T21:25:01,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742068_1244 (size=32107) 2024-11-17T21:25:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T21:25:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T21:25:01,673 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/b4005db4faae4cfba0badb0376784194 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/b4005db4faae4cfba0badb0376784194 2024-11-17T21:25:01,679 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4554a61bae93fe0dedbab5f4c1513837/A of 4554a61bae93fe0dedbab5f4c1513837 into b4005db4faae4cfba0badb0376784194(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:01,679 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4554a61bae93fe0dedbab5f4c1513837: 2024-11-17T21:25:01,679 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:25:01,679 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., storeName=4554a61bae93fe0dedbab5f4c1513837/A, priority=12, startTime=1731878700797; duration=0sec 2024-11-17T21:25:01,679 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:25:01,679 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:01,679 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. after waiting 0 ms 2024-11-17T21:25:01,679 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4554a61bae93fe0dedbab5f4c1513837:A 2024-11-17T21:25:01,679 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:25:01,679 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing 4554a61bae93fe0dedbab5f4c1513837 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-17T21:25:01,685 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111764b5f74a90b94ff2a5b016cd50c24b28_4554a61bae93fe0dedbab5f4c1513837 is 50, key is test_row_0/A:col10/1731878700991/Put/seqid=0 2024-11-17T21:25:01,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742069_1245 (size=7374) 2024-11-17T21:25:02,099 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:02,108 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111764b5f74a90b94ff2a5b016cd50c24b28_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111764b5f74a90b94ff2a5b016cd50c24b28_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:02,109 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/790adb2a4b934c878b2be7210aa44300, store: [table=TestAcidGuarantees family=A region=4554a61bae93fe0dedbab5f4c1513837] 2024-11-17T21:25:02,110 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/790adb2a4b934c878b2be7210aa44300 is 175, key is test_row_0/A:col10/1731878700991/Put/seqid=0 2024-11-17T21:25:02,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742070_1246 (size=13865) 2024-11-17T21:25:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T21:25:02,514 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/790adb2a4b934c878b2be7210aa44300 2024-11-17T21:25:02,522 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/0973e8e1df9a40f1a14859160bba69cc is 50, key is test_row_0/B:col10/1731878700991/Put/seqid=0 2024-11-17T21:25:02,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742071_1247 (size=7415) 2024-11-17T21:25:02,926 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/0973e8e1df9a40f1a14859160bba69cc 2024-11-17T21:25:02,983 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/8422abfff34f40a99ddad12a8e107328 is 50, key is test_row_0/C:col10/1731878700991/Put/seqid=0 2024-11-17T21:25:02,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742072_1248 (size=7415) 2024-11-17T21:25:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T21:25:03,390 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/8422abfff34f40a99ddad12a8e107328 2024-11-17T21:25:03,400 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/A/790adb2a4b934c878b2be7210aa44300 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/790adb2a4b934c878b2be7210aa44300 2024-11-17T21:25:03,406 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/790adb2a4b934c878b2be7210aa44300, entries=50, sequenceid=375, filesize=13.5 K 2024-11-17T21:25:03,407 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/B/0973e8e1df9a40f1a14859160bba69cc as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/0973e8e1df9a40f1a14859160bba69cc 2024-11-17T21:25:03,412 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/0973e8e1df9a40f1a14859160bba69cc, entries=50, sequenceid=375, filesize=7.2 K 2024-11-17T21:25:03,413 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/.tmp/C/8422abfff34f40a99ddad12a8e107328 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/8422abfff34f40a99ddad12a8e107328 2024-11-17T21:25:03,416 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/8422abfff34f40a99ddad12a8e107328, entries=50, sequenceid=375, filesize=7.2 K 2024-11-17T21:25:03,417 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 4554a61bae93fe0dedbab5f4c1513837 in 1738ms, sequenceid=375, compaction requested=true 2024-11-17T21:25:03,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/a2c0426fec834e3cb85b08e11e805513, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/ea476ef4e55b4bec9cd1e1ee7971555b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/c6e0665b8b634e368ad5ea0f1910fe9d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/f4f9b441b6604f57a1a9b6fa132f4159, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/78c07c28cef64c86abfad52ab26b787d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3017606f80b34a74b393ce3ea1838fe5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9fe2e7d33cdd4d3da2c1762468b9dc87, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/f470c50971064114bc6ad21390cca78a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9b4f34b28f8c43eda56e87644431f2c3, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/75d3585afa3e42c3a8423a73d37d0536, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/999fe71bd900436d8392e09e1034020d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/337a41c960a548cd9d61ad89e3d05240, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/d9aeb9552ea04252bb98e480e6dc498a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/beb666c62c3b46f7b496949662526082, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/425d274774014111b81c3eee05d2bf2f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/e208d16c2b404d429693ab751e8e0a68, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/74a70577559a4ab4bc5be928d0a8475e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/47b0e57ed95d424d8e7f0b399199c678, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/ec0941b75d9748ce85c8a72506332fda, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/36c716e10dc14ddbba050b630256707d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/39897bb69837463fa7090803749593ea, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3cb53fd3eab74b8daf3c0d14759e5db7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/c99aecf97f0b4190bf98f2e48682bd61, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3edad44b93744ba19d5a6fb7551a7b81, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9a6bd86963e04fcc89c3e2d2dbeeb9c2] to archive 2024-11-17T21:25:03,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:25:03,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/a2c0426fec834e3cb85b08e11e805513 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/a2c0426fec834e3cb85b08e11e805513 2024-11-17T21:25:03,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/ea476ef4e55b4bec9cd1e1ee7971555b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/ea476ef4e55b4bec9cd1e1ee7971555b 2024-11-17T21:25:03,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/c6e0665b8b634e368ad5ea0f1910fe9d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/c6e0665b8b634e368ad5ea0f1910fe9d 2024-11-17T21:25:03,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/f4f9b441b6604f57a1a9b6fa132f4159 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/f4f9b441b6604f57a1a9b6fa132f4159 2024-11-17T21:25:03,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/78c07c28cef64c86abfad52ab26b787d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/78c07c28cef64c86abfad52ab26b787d 2024-11-17T21:25:03,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3017606f80b34a74b393ce3ea1838fe5 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3017606f80b34a74b393ce3ea1838fe5 2024-11-17T21:25:03,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9fe2e7d33cdd4d3da2c1762468b9dc87 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9fe2e7d33cdd4d3da2c1762468b9dc87 2024-11-17T21:25:03,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/f470c50971064114bc6ad21390cca78a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/f470c50971064114bc6ad21390cca78a 2024-11-17T21:25:03,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9b4f34b28f8c43eda56e87644431f2c3 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9b4f34b28f8c43eda56e87644431f2c3 2024-11-17T21:25:03,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/75d3585afa3e42c3a8423a73d37d0536 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/75d3585afa3e42c3a8423a73d37d0536 2024-11-17T21:25:03,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/999fe71bd900436d8392e09e1034020d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/999fe71bd900436d8392e09e1034020d 2024-11-17T21:25:03,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/337a41c960a548cd9d61ad89e3d05240 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/337a41c960a548cd9d61ad89e3d05240 2024-11-17T21:25:03,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/d9aeb9552ea04252bb98e480e6dc498a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/d9aeb9552ea04252bb98e480e6dc498a 2024-11-17T21:25:03,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/beb666c62c3b46f7b496949662526082 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/beb666c62c3b46f7b496949662526082 2024-11-17T21:25:03,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/425d274774014111b81c3eee05d2bf2f to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/425d274774014111b81c3eee05d2bf2f 2024-11-17T21:25:03,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/e208d16c2b404d429693ab751e8e0a68 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/e208d16c2b404d429693ab751e8e0a68 2024-11-17T21:25:03,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/74a70577559a4ab4bc5be928d0a8475e to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/74a70577559a4ab4bc5be928d0a8475e 2024-11-17T21:25:03,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/47b0e57ed95d424d8e7f0b399199c678 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/47b0e57ed95d424d8e7f0b399199c678 2024-11-17T21:25:03,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/ec0941b75d9748ce85c8a72506332fda to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/ec0941b75d9748ce85c8a72506332fda 2024-11-17T21:25:03,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/36c716e10dc14ddbba050b630256707d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/36c716e10dc14ddbba050b630256707d 2024-11-17T21:25:03,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/39897bb69837463fa7090803749593ea to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/39897bb69837463fa7090803749593ea 2024-11-17T21:25:03,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3cb53fd3eab74b8daf3c0d14759e5db7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3cb53fd3eab74b8daf3c0d14759e5db7 2024-11-17T21:25:03,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/c99aecf97f0b4190bf98f2e48682bd61 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/c99aecf97f0b4190bf98f2e48682bd61 2024-11-17T21:25:03,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3edad44b93744ba19d5a6fb7551a7b81 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/3edad44b93744ba19d5a6fb7551a7b81 2024-11-17T21:25:03,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9a6bd86963e04fcc89c3e2d2dbeeb9c2 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/9a6bd86963e04fcc89c3e2d2dbeeb9c2 2024-11-17T21:25:03,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3617bdf2d7c24a05870219150a8fe0f4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/08bb1d009b3a429b87988616bebe5307, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/1594dc93447f4c9097fffd2f6e2b3d44, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/9f9a2f2513c948eba4f4cd0ae766ace3, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3e99d0c1307f4b2a92e0c912e099df64, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/b7316dfb45e646709ddfc360ef480548, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/93037c3849ed45c1825538aee7307028, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/07a7cff37379463c83e5436e3452ac5d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/0c05cf32726c4422a86ddeab34ae6717, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/51f31251b4614243844de41c49c34b7e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/92bd1421082a42d8a121bc4fe3455617, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/1ea6ee5250504a43a1da6faa82e0b303, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/75fdc8ac69ec4db5b86f9a1a50875ffa, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/8b59104c0c9243b6b408c279b23796cb, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/ab88166947134fa7bbe0653b21a4fbbf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/d9747076379f442c86c8b316fcfe86b4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/b6beded0c5404284b6dd2bfa7ec11f6f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3cad7050f2624c69a6227d876b7f2bfd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/c84f899fff3546bb8f43c64a286cc934, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/05a2fb65886d4b3dbba9fa6730466eae, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/568255adc6dc4a62b15de49067adb8e8, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/11ba5059c1f7401f8bf1e16fcab2de83, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/d05a6ac9d22a4bc09261952afc52ceb4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/c6d33eb8a5b04e829352d15019d7fb0f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/e6f4359c56c246f2b88af9fa4d07582f] to archive 2024-11-17T21:25:03,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:25:03,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3617bdf2d7c24a05870219150a8fe0f4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3617bdf2d7c24a05870219150a8fe0f4 2024-11-17T21:25:03,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/08bb1d009b3a429b87988616bebe5307 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/08bb1d009b3a429b87988616bebe5307 2024-11-17T21:25:03,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/1594dc93447f4c9097fffd2f6e2b3d44 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/1594dc93447f4c9097fffd2f6e2b3d44 2024-11-17T21:25:03,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/9f9a2f2513c948eba4f4cd0ae766ace3 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/9f9a2f2513c948eba4f4cd0ae766ace3 2024-11-17T21:25:03,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3e99d0c1307f4b2a92e0c912e099df64 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3e99d0c1307f4b2a92e0c912e099df64 2024-11-17T21:25:03,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/b7316dfb45e646709ddfc360ef480548 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/b7316dfb45e646709ddfc360ef480548 2024-11-17T21:25:03,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/93037c3849ed45c1825538aee7307028 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/93037c3849ed45c1825538aee7307028 2024-11-17T21:25:03,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/07a7cff37379463c83e5436e3452ac5d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/07a7cff37379463c83e5436e3452ac5d 2024-11-17T21:25:03,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/0c05cf32726c4422a86ddeab34ae6717 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/0c05cf32726c4422a86ddeab34ae6717 2024-11-17T21:25:03,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/51f31251b4614243844de41c49c34b7e to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/51f31251b4614243844de41c49c34b7e 2024-11-17T21:25:03,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/92bd1421082a42d8a121bc4fe3455617 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/92bd1421082a42d8a121bc4fe3455617 2024-11-17T21:25:03,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/1ea6ee5250504a43a1da6faa82e0b303 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/1ea6ee5250504a43a1da6faa82e0b303 2024-11-17T21:25:03,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/75fdc8ac69ec4db5b86f9a1a50875ffa to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/75fdc8ac69ec4db5b86f9a1a50875ffa 2024-11-17T21:25:03,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/8b59104c0c9243b6b408c279b23796cb to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/8b59104c0c9243b6b408c279b23796cb 2024-11-17T21:25:03,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/ab88166947134fa7bbe0653b21a4fbbf to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/ab88166947134fa7bbe0653b21a4fbbf 2024-11-17T21:25:03,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/d9747076379f442c86c8b316fcfe86b4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/d9747076379f442c86c8b316fcfe86b4 2024-11-17T21:25:03,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/b6beded0c5404284b6dd2bfa7ec11f6f to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/b6beded0c5404284b6dd2bfa7ec11f6f 2024-11-17T21:25:03,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3cad7050f2624c69a6227d876b7f2bfd to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/3cad7050f2624c69a6227d876b7f2bfd 2024-11-17T21:25:03,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/c84f899fff3546bb8f43c64a286cc934 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/c84f899fff3546bb8f43c64a286cc934 2024-11-17T21:25:03,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/05a2fb65886d4b3dbba9fa6730466eae to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/05a2fb65886d4b3dbba9fa6730466eae 2024-11-17T21:25:03,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/568255adc6dc4a62b15de49067adb8e8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/568255adc6dc4a62b15de49067adb8e8 2024-11-17T21:25:03,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/11ba5059c1f7401f8bf1e16fcab2de83 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/11ba5059c1f7401f8bf1e16fcab2de83 2024-11-17T21:25:03,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/d05a6ac9d22a4bc09261952afc52ceb4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/d05a6ac9d22a4bc09261952afc52ceb4 2024-11-17T21:25:03,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/c6d33eb8a5b04e829352d15019d7fb0f to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/c6d33eb8a5b04e829352d15019d7fb0f 2024-11-17T21:25:03,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/e6f4359c56c246f2b88af9fa4d07582f to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/e6f4359c56c246f2b88af9fa4d07582f 2024-11-17T21:25:03,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/e215cfb06dc742d68badc5e39e4360c5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/6c57f426be0d429f8727d4fd1d770500, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/084d4f56025444759de9297fd73e13de, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/2c6e80e8862b493198ae90f31c5221c9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/55d929599f2e4600934a9b38da334272, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/d250871dc88a4e238270eb6f3d6d5f8d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/4797818bef6d4ae69b0edac9e459a54d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/7ef3711f215740c7aa1902442da98439, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/67e313311bdb46b7b5450c13d5d46181, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/414e62a1b04a43458378304eb7bc604c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/ab6759bb8fb347449d28e0d3cd585ac1, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/54e25f5fb3a246219ca71bf9df72e2b4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/0708382940a3471588f6884ea742c569, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/1fc5d07c8d444c72aeab1172324f25ab, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/8b92e8acccf942b9a06368caa0e25e84, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/20c3a71bc2644fff928f02ddb2b38cc4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/a65fa05a3a144aab9405380a52e8e7c4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/bf3aa277a26f4919b11cbeb504eca5a9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/30fbc808742141c3b54043d1eea92303, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/438299e1fbba4b38b00bedbb56708843, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/50c49b5b62004cd783933b24de22e646] to archive 2024-11-17T21:25:03,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:25:03,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/e215cfb06dc742d68badc5e39e4360c5 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/e215cfb06dc742d68badc5e39e4360c5 2024-11-17T21:25:03,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/6c57f426be0d429f8727d4fd1d770500 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/6c57f426be0d429f8727d4fd1d770500 2024-11-17T21:25:03,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/084d4f56025444759de9297fd73e13de to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/084d4f56025444759de9297fd73e13de 2024-11-17T21:25:03,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/2c6e80e8862b493198ae90f31c5221c9 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/2c6e80e8862b493198ae90f31c5221c9 2024-11-17T21:25:03,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/55d929599f2e4600934a9b38da334272 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/55d929599f2e4600934a9b38da334272 2024-11-17T21:25:03,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/d250871dc88a4e238270eb6f3d6d5f8d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/d250871dc88a4e238270eb6f3d6d5f8d 2024-11-17T21:25:03,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/4797818bef6d4ae69b0edac9e459a54d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/4797818bef6d4ae69b0edac9e459a54d 2024-11-17T21:25:03,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/7ef3711f215740c7aa1902442da98439 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/7ef3711f215740c7aa1902442da98439 2024-11-17T21:25:03,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/67e313311bdb46b7b5450c13d5d46181 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/67e313311bdb46b7b5450c13d5d46181 2024-11-17T21:25:03,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/414e62a1b04a43458378304eb7bc604c to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/414e62a1b04a43458378304eb7bc604c 2024-11-17T21:25:03,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/ab6759bb8fb347449d28e0d3cd585ac1 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/ab6759bb8fb347449d28e0d3cd585ac1 2024-11-17T21:25:03,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/54e25f5fb3a246219ca71bf9df72e2b4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/54e25f5fb3a246219ca71bf9df72e2b4 2024-11-17T21:25:03,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/0708382940a3471588f6884ea742c569 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/0708382940a3471588f6884ea742c569 2024-11-17T21:25:03,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/1fc5d07c8d444c72aeab1172324f25ab to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/1fc5d07c8d444c72aeab1172324f25ab 2024-11-17T21:25:03,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/8b92e8acccf942b9a06368caa0e25e84 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/8b92e8acccf942b9a06368caa0e25e84 2024-11-17T21:25:03,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/20c3a71bc2644fff928f02ddb2b38cc4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/20c3a71bc2644fff928f02ddb2b38cc4 2024-11-17T21:25:03,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/a65fa05a3a144aab9405380a52e8e7c4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/a65fa05a3a144aab9405380a52e8e7c4 2024-11-17T21:25:03,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/bf3aa277a26f4919b11cbeb504eca5a9 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/bf3aa277a26f4919b11cbeb504eca5a9 2024-11-17T21:25:03,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/30fbc808742141c3b54043d1eea92303 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/30fbc808742141c3b54043d1eea92303 2024-11-17T21:25:03,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/438299e1fbba4b38b00bedbb56708843 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/438299e1fbba4b38b00bedbb56708843 2024-11-17T21:25:03,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/50c49b5b62004cd783933b24de22e646 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/50c49b5b62004cd783933b24de22e646 2024-11-17T21:25:03,492 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/recovered.edits/378.seqid, newMaxSeqId=378, maxSeqId=4 2024-11-17T21:25:03,493 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. 2024-11-17T21:25:03,493 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for 4554a61bae93fe0dedbab5f4c1513837: Waiting for close lock at 1731878701160Running coprocessor pre-close hooks at 1731878701160Disabling compacts and flushes for region at 1731878701160Disabling writes for close at 1731878701679 (+519 ms)Obtaining lock to block concurrent updates at 1731878701679Preparing flush snapshotting stores in 4554a61bae93fe0dedbab5f4c1513837 at 1731878701679Finished memstore snapshotting TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731878701679Flushing stores of TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837. at 1731878701680 (+1 ms)Flushing 4554a61bae93fe0dedbab5f4c1513837/A: creating writer at 1731878701680Flushing mob file 4554a61bae93fe0dedbab5f4c1513837/A: appending metadata at 1731878701685 (+5 ms)Flushing mob file 4554a61bae93fe0dedbab5f4c1513837/A: closing flushed file at 1731878701685Flushing 4554a61bae93fe0dedbab5f4c1513837/A: appending metadata at 1731878702109 (+424 ms)Flushing 4554a61bae93fe0dedbab5f4c1513837/A: closing flushed file at 1731878702109Flushing 4554a61bae93fe0dedbab5f4c1513837/B: creating writer at 1731878702518 (+409 ms)Flushing 4554a61bae93fe0dedbab5f4c1513837/B: appending metadata at 1731878702521 (+3 ms)Flushing 4554a61bae93fe0dedbab5f4c1513837/B: closing flushed file at 1731878702521Flushing 4554a61bae93fe0dedbab5f4c1513837/C: creating writer at 1731878702980 (+459 ms)Flushing 4554a61bae93fe0dedbab5f4c1513837/C: appending metadata at 1731878702983 (+3 ms)Flushing 4554a61bae93fe0dedbab5f4c1513837/C: closing flushed file at 1731878702983Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1aa2a456: reopening flushed file at 1731878703399 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3044f762: reopening flushed file at 1731878703406 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2db2d212: reopening flushed file at 1731878703412 (+6 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 4554a61bae93fe0dedbab5f4c1513837 in 1738ms, sequenceid=375, compaction requested=true at 1731878703417 (+5 ms)Writing region close event to WAL at 1731878703489 (+72 ms)Running coprocessor post-close hooks at 1731878703493 (+4 ms)Closed at 1731878703493 2024-11-17T21:25:03,495 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed 4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:03,496 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=4554a61bae93fe0dedbab5f4c1513837, regionState=CLOSED 2024-11-17T21:25:03,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:25:03,499 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-17T21:25:03,499 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure 4554a61bae93fe0dedbab5f4c1513837, server=a313eea8709e,46445,1731878642828 in 2.4910 sec 2024-11-17T21:25:03,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-17T21:25:03,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4554a61bae93fe0dedbab5f4c1513837, UNASSIGN in 2.4960 sec 2024-11-17T21:25:03,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-17T21:25:03,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.4980 sec 2024-11-17T21:25:03,502 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878703502"}]},"ts":"1731878703502"} 2024-11-17T21:25:03,503 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T21:25:03,503 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T21:25:03,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5080 sec 2024-11-17T21:25:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T21:25:05,161 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:05,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T21:25:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:05,163 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T21:25:05,163 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:05,165 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,168 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/recovered.edits] 2024-11-17T21:25:05,170 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/790adb2a4b934c878b2be7210aa44300 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/790adb2a4b934c878b2be7210aa44300 2024-11-17T21:25:05,171 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/b4005db4faae4cfba0badb0376784194 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/A/b4005db4faae4cfba0badb0376784194 2024-11-17T21:25:05,173 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/0973e8e1df9a40f1a14859160bba69cc to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/0973e8e1df9a40f1a14859160bba69cc 2024-11-17T21:25:05,173 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/a54200aaddba4d92bf9ea750aaa7922c to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/B/a54200aaddba4d92bf9ea750aaa7922c 2024-11-17T21:25:05,175 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/310d8113d07d4b85b3f5f9e4207157ff to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/310d8113d07d4b85b3f5f9e4207157ff 2024-11-17T21:25:05,176 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/4f7f5a185ddd419caa978b8250e88405 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/4f7f5a185ddd419caa978b8250e88405 2024-11-17T21:25:05,177 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/66d48c45ba964c0d8d873da4ccbbb8a9 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/66d48c45ba964c0d8d873da4ccbbb8a9 2024-11-17T21:25:05,177 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/8422abfff34f40a99ddad12a8e107328 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/8422abfff34f40a99ddad12a8e107328 2024-11-17T21:25:05,178 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/a31bcf78134a4c81bfac85ef11cd29a9 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/C/a31bcf78134a4c81bfac85ef11cd29a9 2024-11-17T21:25:05,180 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/recovered.edits/378.seqid to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837/recovered.edits/378.seqid 2024-11-17T21:25:05,181 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,181 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T21:25:05,181 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T21:25:05,182 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T21:25:05,185 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170e86d88a78ec498abe7902c459402ee3_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170e86d88a78ec498abe7902c459402ee3_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,186 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173d4d0c1f4be648bdbf813ce2283b9a86_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173d4d0c1f4be648bdbf813ce2283b9a86_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,187 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111745a1e164c48a4fa2b7fa65ae8dce22a0_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111745a1e164c48a4fa2b7fa65ae8dce22a0_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,188 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117559fd49f92e44ba0b928a054d4e5a9cb_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117559fd49f92e44ba0b928a054d4e5a9cb_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,189 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175a3beada52614facb773ee8f7f323cd7_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175a3beada52614facb773ee8f7f323cd7_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,191 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117607bf69b2d32490fbb69186327de0b28_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117607bf69b2d32490fbb69186327de0b28_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,193 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111764b5f74a90b94ff2a5b016cd50c24b28_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111764b5f74a90b94ff2a5b016cd50c24b28_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,195 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111769c1b0c0201349179858a048035bc797_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111769c1b0c0201349179858a048035bc797_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,196 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111777fb37d0cb474e4689ab472a21320b7f_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111777fb37d0cb474e4689ab472a21320b7f_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,197 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a06830281ce14e33a5d466ad6586bc01_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a06830281ce14e33a5d466ad6586bc01_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,199 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a1e015e9728547419cbfe29583e92b74_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a1e015e9728547419cbfe29583e92b74_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,200 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a614c36e595a4d9c9954fddce203029d_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a614c36e595a4d9c9954fddce203029d_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,202 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a622a227d0624585bc253bda95d1f1ed_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a622a227d0624585bc253bda95d1f1ed_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,203 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a6b83f50602447a39ee9bc9a9f19833a_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a6b83f50602447a39ee9bc9a9f19833a_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,204 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b29e568de9774c42a3cd96d0b19964df_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b29e568de9774c42a3cd96d0b19964df_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,206 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b4dc34af043f412dbd3f8295fd336f9e_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b4dc34af043f412dbd3f8295fd336f9e_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,207 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b5eba6729a914011b92c7a332fad70eb_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b5eba6729a914011b92c7a332fad70eb_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,208 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b8f27a56ff954dc68332759e7f4f4c0f_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b8f27a56ff954dc68332759e7f4f4c0f_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,210 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c6ad99929bb34d478a8533381c8cef12_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c6ad99929bb34d478a8533381c8cef12_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,211 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f15a4a6c79064e71b82eed19959c22ba_4554a61bae93fe0dedbab5f4c1513837 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f15a4a6c79064e71b82eed19959c22ba_4554a61bae93fe0dedbab5f4c1513837 2024-11-17T21:25:05,212 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T21:25:05,214 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:05,217 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T21:25:05,219 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T21:25:05,221 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:05,221 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T21:25:05,221 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731878705221"}]},"ts":"9223372036854775807"} 2024-11-17T21:25:05,224 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T21:25:05,224 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4554a61bae93fe0dedbab5f4c1513837, NAME => 'TestAcidGuarantees,,1731878674351.4554a61bae93fe0dedbab5f4c1513837.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T21:25:05,224 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T21:25:05,224 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731878705224"}]},"ts":"9223372036854775807"} 2024-11-17T21:25:05,226 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T21:25:05,227 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:05,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 66 msec 2024-11-17T21:25:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T21:25:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T21:25:05,271 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:05,278 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=241 (was 240) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/cluster_6acd51c4-a1e3-75e6-757b-0efaa889834a/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_58963030_22 at /127.0.0.1:51738 [Waiting for operation #247] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1395276899_22 at /127.0.0.1:35476 [Waiting for operation #729] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1395276899_22 at /127.0.0.1:51728 [Waiting for operation #227] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_58963030_22 at /127.0.0.1:51458 [Waiting for operation #306] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/cluster_6acd51c4-a1e3-75e6-757b-0efaa889834a/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=451 (was 439) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8161 (was 8710) 2024-11-17T21:25:05,285 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=451, ProcessCount=11, AvailableMemoryMB=8160 2024-11-17T21:25:05,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T21:25:05,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T21:25:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:05,288 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T21:25:05,288 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:05,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-17T21:25:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-17T21:25:05,289 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T21:25:05,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742073_1249 (size=959) 2024-11-17T21:25:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-17T21:25:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-17T21:25:05,696 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249 2024-11-17T21:25:05,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742074_1250 (size=53) 2024-11-17T21:25:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-17T21:25:06,101 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:25:06,101 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 224e951b47a00740dfea2342be622072, disabling compactions & flushes 2024-11-17T21:25:06,101 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:06,102 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:06,102 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. after waiting 0 ms 2024-11-17T21:25:06,102 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:06,102 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:06,102 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 224e951b47a00740dfea2342be622072: Waiting for close lock at 1731878706101Disabling compacts and flushes for region at 1731878706101Disabling writes for close at 1731878706102 (+1 ms)Writing region close event to WAL at 1731878706102Closed at 1731878706102 2024-11-17T21:25:06,103 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T21:25:06,103 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731878706103"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731878706103"}]},"ts":"1731878706103"} 2024-11-17T21:25:06,105 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T21:25:06,106 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T21:25:06,106 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878706106"}]},"ts":"1731878706106"} 2024-11-17T21:25:06,108 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T21:25:06,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=224e951b47a00740dfea2342be622072, ASSIGN}] 2024-11-17T21:25:06,109 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=224e951b47a00740dfea2342be622072, ASSIGN 2024-11-17T21:25:06,110 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=224e951b47a00740dfea2342be622072, ASSIGN; state=OFFLINE, location=a313eea8709e,46445,1731878642828; forceNewPlan=false, retain=false 2024-11-17T21:25:06,261 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=224e951b47a00740dfea2342be622072, regionState=OPENING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:25:06,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=224e951b47a00740dfea2342be622072, ASSIGN because future has completed 2024-11-17T21:25:06,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:25:06,418 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:06,419 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} 2024-11-17T21:25:06,419 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 224e951b47a00740dfea2342be622072 2024-11-17T21:25:06,419 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:25:06,419 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for 224e951b47a00740dfea2342be622072 2024-11-17T21:25:06,419 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for 224e951b47a00740dfea2342be622072 2024-11-17T21:25:06,421 INFO [StoreOpener-224e951b47a00740dfea2342be622072-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 224e951b47a00740dfea2342be622072 2024-11-17T21:25:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-17T21:25:06,423 INFO [StoreOpener-224e951b47a00740dfea2342be622072-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 224e951b47a00740dfea2342be622072 columnFamilyName A 2024-11-17T21:25:06,423 DEBUG [StoreOpener-224e951b47a00740dfea2342be622072-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:06,424 INFO [StoreOpener-224e951b47a00740dfea2342be622072-1 {}] regionserver.HStore(327): Store=224e951b47a00740dfea2342be622072/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:25:06,424 INFO [StoreOpener-224e951b47a00740dfea2342be622072-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 224e951b47a00740dfea2342be622072 2024-11-17T21:25:06,425 INFO [StoreOpener-224e951b47a00740dfea2342be622072-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 224e951b47a00740dfea2342be622072 columnFamilyName B 2024-11-17T21:25:06,425 DEBUG [StoreOpener-224e951b47a00740dfea2342be622072-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:06,426 INFO [StoreOpener-224e951b47a00740dfea2342be622072-1 {}] regionserver.HStore(327): Store=224e951b47a00740dfea2342be622072/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:25:06,426 INFO [StoreOpener-224e951b47a00740dfea2342be622072-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 224e951b47a00740dfea2342be622072 2024-11-17T21:25:06,427 INFO [StoreOpener-224e951b47a00740dfea2342be622072-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 224e951b47a00740dfea2342be622072 columnFamilyName C 2024-11-17T21:25:06,427 DEBUG [StoreOpener-224e951b47a00740dfea2342be622072-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:06,428 INFO [StoreOpener-224e951b47a00740dfea2342be622072-1 {}] regionserver.HStore(327): Store=224e951b47a00740dfea2342be622072/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:25:06,428 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for 224e951b47a00740dfea2342be622072 2024-11-17T21:25:06,429 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072 2024-11-17T21:25:06,429 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072 2024-11-17T21:25:06,430 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for 224e951b47a00740dfea2342be622072 2024-11-17T21:25:06,430 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for 224e951b47a00740dfea2342be622072 2024-11-17T21:25:06,431 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T21:25:06,432 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for 224e951b47a00740dfea2342be622072 2024-11-17T21:25:06,434 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T21:25:06,434 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened 224e951b47a00740dfea2342be622072; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67062008, jitterRate=-6.982088088989258E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T21:25:06,435 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 224e951b47a00740dfea2342be622072 2024-11-17T21:25:06,436 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for 224e951b47a00740dfea2342be622072: Running coprocessor pre-open hook at 1731878706419Writing region info on filesystem at 1731878706419Initializing all the Stores at 1731878706420 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878706420Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878706420Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878706420Cleaning up temporary data from old regions at 1731878706430 (+10 ms)Running coprocessor post-open hooks at 1731878706435 (+5 ms)Region opened successfully at 1731878706436 (+1 ms) 2024-11-17T21:25:06,437 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., pid=67, masterSystemTime=1731878706414 2024-11-17T21:25:06,440 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:06,440 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:06,441 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=224e951b47a00740dfea2342be622072, regionState=OPEN, openSeqNum=2, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:25:06,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:25:06,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-17T21:25:06,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 in 183 msec 2024-11-17T21:25:06,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-17T21:25:06,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=224e951b47a00740dfea2342be622072, ASSIGN in 339 msec 2024-11-17T21:25:06,450 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T21:25:06,451 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878706450"}]},"ts":"1731878706450"} 2024-11-17T21:25:06,452 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T21:25:06,454 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T21:25:06,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1680 sec 2024-11-17T21:25:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-17T21:25:07,441 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:07,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13a2acf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,442 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:07,442 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:07,443 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:07,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:07,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@86d9baa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:07,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:07,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:07,445 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:07,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@294d74df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,446 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:07,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:07,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:07,448 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T21:25:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is a313eea8709e,43623,1731878641945 2024-11-17T21:25:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6ec5b3a0 2024-11-17T21:25:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T21:25:07,450 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T21:25:07,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@661204ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,451 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:07,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:07,452 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8d61fd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:07,454 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:07,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2614c573, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40655b61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:07,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:07,456 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3208cda0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:07,457 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:07,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c331b2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d355d6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,458 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:07,458 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:07,459 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:07,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:07,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:07,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66da3a51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:07,461 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf8203f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf4401, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,462 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:07,462 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:07,463 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bb1878, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:07,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:07,464 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3720ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d01f46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,466 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:07,466 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:07,467 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2111a97a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:07,469 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f7041ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a4b43fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,470 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:07,470 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:07,471 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:07,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:07,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:07,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c2becfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:07,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:07,472 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14202d36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aa78cca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,474 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:07,474 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:07,475 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54135bcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:07,476 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18d29809, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d06ee61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,477 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:07,477 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:07,478 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2b25e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:07,480 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c06292, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a31e52b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,481 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:07,481 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:07,482 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd53682, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:07,483 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@651a38f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ed5ef25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,485 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:07,485 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:07,486 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a7934ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:07,487 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42632f20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:07,489 DEBUG [Thread-1166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:07,489 DEBUG [Thread-1162 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:07,489 DEBUG [Thread-1160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:07,489 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:25:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:07,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:07,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:07,490 DEBUG [Thread-1171 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-17T21:25:07,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:07,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T21:25:07,491 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:07,491 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:25:07,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:07,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:07,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:07,492 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:25:07,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:25:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:25:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:25:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:25:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:25:07,495 DEBUG [Thread-1173 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:07,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:07,496 DEBUG [Thread-1175 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:07,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:07,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:07,500 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:25:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:25:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:07,508 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:07,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T21:25:07,512 DEBUG [Thread-1177 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:07,514 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:07,515 DEBUG [Thread-1179 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:07,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:07,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:25:07,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:07,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:07,520 DEBUG [Thread-1164 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:07,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:07,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:07,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:07,523 DEBUG [Thread-1168 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:07,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:07,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:25:07,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:07,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:25:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:25:07,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/a6c4f7330a1f4f1a965ad8ec17ffdf77 is 50, key is test_row_0/A:col10/1731878707505/Put/seqid=0 2024-11-17T21:25:07,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878767550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:07,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:07,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878767551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:07,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:07,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:07,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878767551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:07,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:07,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878767553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:07,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878767558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:07,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:07,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742075_1251 (size=12001) 2024-11-17T21:25:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T21:25:07,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T21:25:07,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:07,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:07,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:07,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:07,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:07,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878767664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:07,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878767664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:07,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878767665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:07,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878767671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:07,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878767671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:07,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T21:25:07,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:07,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:07,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:07,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:07,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:07,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:07,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T21:25:07,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:07,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878767871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:07,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:07,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878767881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:07,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878767881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:07,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878767881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:07,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878767881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:07,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T21:25:07,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:07,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:07,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:07,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:07,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:07,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/a6c4f7330a1f4f1a965ad8ec17ffdf77 2024-11-17T21:25:08,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/46b4a616bea942e192ea587bbcdc7779 is 50, key is test_row_0/B:col10/1731878707505/Put/seqid=0 2024-11-17T21:25:08,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742076_1252 (size=12001) 2024-11-17T21:25:08,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/46b4a616bea942e192ea587bbcdc7779 2024-11-17T21:25:08,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/4904d35a5938494b8957e6d59313b1c8 is 50, key is test_row_0/C:col10/1731878707505/Put/seqid=0 2024-11-17T21:25:08,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742077_1253 (size=12001) 2024-11-17T21:25:08,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/4904d35a5938494b8957e6d59313b1c8 2024-11-17T21:25:08,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/a6c4f7330a1f4f1a965ad8ec17ffdf77 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/a6c4f7330a1f4f1a965ad8ec17ffdf77 2024-11-17T21:25:08,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/a6c4f7330a1f4f1a965ad8ec17ffdf77, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T21:25:08,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/46b4a616bea942e192ea587bbcdc7779 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/46b4a616bea942e192ea587bbcdc7779 2024-11-17T21:25:08,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/46b4a616bea942e192ea587bbcdc7779, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T21:25:08,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/4904d35a5938494b8957e6d59313b1c8 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/4904d35a5938494b8957e6d59313b1c8 2024-11-17T21:25:08,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T21:25:08,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:08,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:08,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:08,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/4904d35a5938494b8957e6d59313b1c8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T21:25:08,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 224e951b47a00740dfea2342be622072 in 608ms, sequenceid=13, compaction requested=false 2024-11-17T21:25:08,116 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T21:25:08,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T21:25:08,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:08,184 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:08,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T21:25:08,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878768198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878768198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878768199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878768202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878768207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/cc092080ab514ca2a18c90a3eedcfabf is 50, key is test_row_0/A:col10/1731878707549/Put/seqid=0 2024-11-17T21:25:08,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742078_1254 (size=14341) 2024-11-17T21:25:08,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T21:25:08,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:08,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:08,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:08,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878768311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878768311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878768321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878768331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878768331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T21:25:08,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:08,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:08,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:08,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878768521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878768521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878768531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878768541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878768541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T21:25:08,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:08,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:08,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:08,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T21:25:08,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/cc092080ab514ca2a18c90a3eedcfabf 2024-11-17T21:25:08,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/413abd33719540829c22cd3c09873faf is 50, key is test_row_0/B:col10/1731878707549/Put/seqid=0 2024-11-17T21:25:08,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742079_1255 (size=12001) 2024-11-17T21:25:08,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/413abd33719540829c22cd3c09873faf 2024-11-17T21:25:08,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/88955ed64cfa4abe8dd1181a66d72be3 is 50, key is test_row_0/C:col10/1731878707549/Put/seqid=0 2024-11-17T21:25:08,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T21:25:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:08,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742080_1256 (size=12001) 2024-11-17T21:25:08,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/88955ed64cfa4abe8dd1181a66d72be3 2024-11-17T21:25:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/cc092080ab514ca2a18c90a3eedcfabf as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/cc092080ab514ca2a18c90a3eedcfabf 2024-11-17T21:25:08,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/cc092080ab514ca2a18c90a3eedcfabf, entries=200, sequenceid=38, filesize=14.0 K 2024-11-17T21:25:08,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/413abd33719540829c22cd3c09873faf as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/413abd33719540829c22cd3c09873faf 2024-11-17T21:25:08,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/413abd33719540829c22cd3c09873faf, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T21:25:08,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/88955ed64cfa4abe8dd1181a66d72be3 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/88955ed64cfa4abe8dd1181a66d72be3 2024-11-17T21:25:08,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/88955ed64cfa4abe8dd1181a66d72be3, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T21:25:08,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 224e951b47a00740dfea2342be622072 in 595ms, sequenceid=38, compaction requested=false 2024-11-17T21:25:08,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:08,832 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:08,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T21:25:08,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/ebd124ff6fec4060890a81c2baac7d72 is 50, key is test_row_0/A:col10/1731878708831/Put/seqid=0 2024-11-17T21:25:08,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T21:25:08,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:08,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:08,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:08,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878768872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878768873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878768873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878768880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878768881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742081_1257 (size=12001) 2024-11-17T21:25:08,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878768991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878768991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878768991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878768991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:08,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878768991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:08,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:08,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:09,021 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T21:25:09,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T21:25:09,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:09,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:09,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:09,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:09,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:09,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T21:25:09,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:09,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:09,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:09,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:09,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:09,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878769201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:09,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878769201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:09,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878769201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:09,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878769201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:09,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878769211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:09,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/ebd124ff6fec4060890a81c2baac7d72 2024-11-17T21:25:09,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/e69148efd2c743249f7923407aba3f25 is 50, key is test_row_0/B:col10/1731878708831/Put/seqid=0 2024-11-17T21:25:09,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742082_1258 (size=12001) 2024-11-17T21:25:09,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T21:25:09,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:09,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:09,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:09,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:09,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:09,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T21:25:09,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:09,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:09,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:09,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:09,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:09,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878769511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:09,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:09,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878769511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:09,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878769511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878769511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:09,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878769521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T21:25:09,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T21:25:09,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:09,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:09,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:09,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:09,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:09,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/e69148efd2c743249f7923407aba3f25 2024-11-17T21:25:09,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/5b3200fa26ff40ed875084026b82e386 is 50, key is test_row_0/C:col10/1731878708831/Put/seqid=0 2024-11-17T21:25:09,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742083_1259 (size=12001) 2024-11-17T21:25:09,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/5b3200fa26ff40ed875084026b82e386 2024-11-17T21:25:09,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/ebd124ff6fec4060890a81c2baac7d72 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/ebd124ff6fec4060890a81c2baac7d72 2024-11-17T21:25:09,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/ebd124ff6fec4060890a81c2baac7d72, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T21:25:09,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/e69148efd2c743249f7923407aba3f25 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e69148efd2c743249f7923407aba3f25 2024-11-17T21:25:09,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e69148efd2c743249f7923407aba3f25, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T21:25:09,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/5b3200fa26ff40ed875084026b82e386 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/5b3200fa26ff40ed875084026b82e386 2024-11-17T21:25:09,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/5b3200fa26ff40ed875084026b82e386, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T21:25:09,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 224e951b47a00740dfea2342be622072 in 905ms, sequenceid=51, compaction requested=true 2024-11-17T21:25:09,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:09,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:25:09,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:09,738 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:09,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:09,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:09,738 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:09,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:25:09,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:09,740 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:09,740 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/B is initiating minor compaction (all files) 2024-11-17T21:25:09,740 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/B in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:09,740 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/46b4a616bea942e192ea587bbcdc7779, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/413abd33719540829c22cd3c09873faf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e69148efd2c743249f7923407aba3f25] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=35.2 K 2024-11-17T21:25:09,740 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:09,740 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/A is initiating minor compaction (all files) 2024-11-17T21:25:09,740 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/A in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:09,740 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/a6c4f7330a1f4f1a965ad8ec17ffdf77, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/cc092080ab514ca2a18c90a3eedcfabf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/ebd124ff6fec4060890a81c2baac7d72] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=37.4 K 2024-11-17T21:25:09,740 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6c4f7330a1f4f1a965ad8ec17ffdf77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731878707500 2024-11-17T21:25:09,741 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 46b4a616bea942e192ea587bbcdc7779, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731878707500 2024-11-17T21:25:09,741 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc092080ab514ca2a18c90a3eedcfabf, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731878707549 2024-11-17T21:25:09,741 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 413abd33719540829c22cd3c09873faf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731878707549 2024-11-17T21:25:09,741 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebd124ff6fec4060890a81c2baac7d72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878708196 2024-11-17T21:25:09,741 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting e69148efd2c743249f7923407aba3f25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878708196 2024-11-17T21:25:09,749 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#A#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:09,749 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#B#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:09,750 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/85d8a1ff24364e349e69888eca1e4387 is 50, key is test_row_0/A:col10/1731878708831/Put/seqid=0 2024-11-17T21:25:09,750 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/809fcb2b54c84a4d92a6657da549831c is 50, key is test_row_0/B:col10/1731878708831/Put/seqid=0 2024-11-17T21:25:09,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742085_1261 (size=12104) 2024-11-17T21:25:09,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742084_1260 (size=12104) 2024-11-17T21:25:09,782 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/809fcb2b54c84a4d92a6657da549831c as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/809fcb2b54c84a4d92a6657da549831c 2024-11-17T21:25:09,787 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 224e951b47a00740dfea2342be622072/B of 224e951b47a00740dfea2342be622072 into 809fcb2b54c84a4d92a6657da549831c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:09,787 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:09,787 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/B, priority=13, startTime=1731878709738; duration=0sec 2024-11-17T21:25:09,787 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:09,787 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:B 2024-11-17T21:25:09,787 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:09,789 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:09,789 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/C is initiating minor compaction (all files) 2024-11-17T21:25:09,789 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/C in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:09,789 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/4904d35a5938494b8957e6d59313b1c8, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/88955ed64cfa4abe8dd1181a66d72be3, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/5b3200fa26ff40ed875084026b82e386] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=35.2 K 2024-11-17T21:25:09,789 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 4904d35a5938494b8957e6d59313b1c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731878707500 2024-11-17T21:25:09,790 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 88955ed64cfa4abe8dd1181a66d72be3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731878707549 2024-11-17T21:25:09,790 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b3200fa26ff40ed875084026b82e386, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878708196 2024-11-17T21:25:09,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T21:25:09,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:09,798 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T21:25:09,798 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#C#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:09,799 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/7f441302a8eb4edc9a1594c88c3f4990 is 50, key is test_row_0/C:col10/1731878708831/Put/seqid=0 2024-11-17T21:25:09,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/9a9d9763d7324207ae25d14b566c838a is 50, key is test_row_0/A:col10/1731878708872/Put/seqid=0 2024-11-17T21:25:09,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742086_1262 (size=12104) 2024-11-17T21:25:09,836 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/7f441302a8eb4edc9a1594c88c3f4990 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/7f441302a8eb4edc9a1594c88c3f4990 2024-11-17T21:25:09,843 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 224e951b47a00740dfea2342be622072/C of 224e951b47a00740dfea2342be622072 into 7f441302a8eb4edc9a1594c88c3f4990(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:09,844 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:09,845 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/C, priority=13, startTime=1731878709738; duration=0sec 2024-11-17T21:25:09,845 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:09,845 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:C 2024-11-17T21:25:09,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742087_1263 (size=12001) 2024-11-17T21:25:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:10,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:10,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878770029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878770029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878770030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878770031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878770033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878770141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878770141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878770141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878770141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,177 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/85d8a1ff24364e349e69888eca1e4387 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/85d8a1ff24364e349e69888eca1e4387 2024-11-17T21:25:10,182 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 224e951b47a00740dfea2342be622072/A of 224e951b47a00740dfea2342be622072 into 85d8a1ff24364e349e69888eca1e4387(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:10,182 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:10,182 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/A, priority=13, startTime=1731878709738; duration=0sec 2024-11-17T21:25:10,182 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:10,182 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:A 2024-11-17T21:25:10,259 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/9a9d9763d7324207ae25d14b566c838a 2024-11-17T21:25:10,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/8e0e33ac0ddb4553b6da997ae28869ef is 50, key is test_row_0/B:col10/1731878708872/Put/seqid=0 2024-11-17T21:25:10,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742088_1264 (size=12001) 2024-11-17T21:25:10,272 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/8e0e33ac0ddb4553b6da997ae28869ef 2024-11-17T21:25:10,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/b445dc700b844eaab91f7559a66ebfa9 is 50, key is test_row_0/C:col10/1731878708872/Put/seqid=0 2024-11-17T21:25:10,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742089_1265 (size=12001) 2024-11-17T21:25:10,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878770351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878770352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878770352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878770352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878770661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878770661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878770661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878770661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:10,685 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/b445dc700b844eaab91f7559a66ebfa9 2024-11-17T21:25:10,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/9a9d9763d7324207ae25d14b566c838a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/9a9d9763d7324207ae25d14b566c838a 2024-11-17T21:25:10,696 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/9a9d9763d7324207ae25d14b566c838a, entries=150, sequenceid=76, filesize=11.7 K 2024-11-17T21:25:10,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/8e0e33ac0ddb4553b6da997ae28869ef as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/8e0e33ac0ddb4553b6da997ae28869ef 2024-11-17T21:25:10,705 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/8e0e33ac0ddb4553b6da997ae28869ef, entries=150, sequenceid=76, filesize=11.7 K 2024-11-17T21:25:10,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/b445dc700b844eaab91f7559a66ebfa9 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/b445dc700b844eaab91f7559a66ebfa9 2024-11-17T21:25:10,713 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/b445dc700b844eaab91f7559a66ebfa9, entries=150, sequenceid=76, filesize=11.7 K 2024-11-17T21:25:10,714 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 224e951b47a00740dfea2342be622072 in 916ms, sequenceid=76, compaction requested=false 2024-11-17T21:25:10,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:10,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:10,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-17T21:25:10,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-17T21:25:10,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-17T21:25:10,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2240 sec 2024-11-17T21:25:10,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 3.2290 sec 2024-11-17T21:25:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:11,042 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:11,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T21:25:11,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/8566340f7900401e9038a52f67cb2761 is 50, key is test_row_0/A:col10/1731878711041/Put/seqid=0 2024-11-17T21:25:11,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742090_1266 (size=12001) 2024-11-17T21:25:11,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878771119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:11,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878771171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:11,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878771171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:11,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878771171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:11,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878771172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:11,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878771231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:11,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:11,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:11,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:11,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878771441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:11,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/8566340f7900401e9038a52f67cb2761 2024-11-17T21:25:11,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/7bceb408522c41e0af49f97bcc429e12 is 50, key is test_row_0/B:col10/1731878711041/Put/seqid=0 2024-11-17T21:25:11,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742091_1267 (size=12001) 2024-11-17T21:25:11,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T21:25:11,661 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:11,661 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:25:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-17T21:25:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T21:25:11,664 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:25:11,665 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:25:11,665 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:25:11,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878771751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T21:25:11,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T21:25:11,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:11,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:11,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:11,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:11,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:11,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/7bceb408522c41e0af49f97bcc429e12 2024-11-17T21:25:11,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/504166cda3ab4106bde8ce646679b909 is 50, key is test_row_0/C:col10/1731878711041/Put/seqid=0 2024-11-17T21:25:11,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742092_1268 (size=12001) 2024-11-17T21:25:11,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T21:25:11,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:11,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:11,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:11,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:11,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T21:25:12,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T21:25:12,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:12,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:12,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:12,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:12,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:12,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878772181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:12,183 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_2' in 224e951b47a00740dfea2342be622072 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:12,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878772191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:12,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878772191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:12,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:12,195 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in 224e951b47a00740dfea2342be622072 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878772191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:12,195 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_2' in 224e951b47a00740dfea2342be622072 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:12,195 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in 224e951b47a00740dfea2342be622072 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:12,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878772261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:12,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T21:25:12,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:12,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:12,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:12,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:12,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T21:25:12,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/504166cda3ab4106bde8ce646679b909 2024-11-17T21:25:12,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/8566340f7900401e9038a52f67cb2761 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8566340f7900401e9038a52f67cb2761 2024-11-17T21:25:12,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8566340f7900401e9038a52f67cb2761, entries=150, sequenceid=91, filesize=11.7 K 2024-11-17T21:25:12,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/7bceb408522c41e0af49f97bcc429e12 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7bceb408522c41e0af49f97bcc429e12 2024-11-17T21:25:12,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7bceb408522c41e0af49f97bcc429e12, entries=150, sequenceid=91, filesize=11.7 K 2024-11-17T21:25:12,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/504166cda3ab4106bde8ce646679b909 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/504166cda3ab4106bde8ce646679b909 2024-11-17T21:25:12,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/504166cda3ab4106bde8ce646679b909, entries=150, sequenceid=91, filesize=11.7 K 2024-11-17T21:25:12,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 224e951b47a00740dfea2342be622072 in 1283ms, sequenceid=91, compaction requested=true 2024-11-17T21:25:12,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:12,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:25:12,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:12,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:12,325 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:12,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:12,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:25:12,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T21:25:12,325 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:12,327 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:12,327 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/B is initiating minor compaction (all files) 2024-11-17T21:25:12,327 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/B in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:12,327 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/809fcb2b54c84a4d92a6657da549831c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/8e0e33ac0ddb4553b6da997ae28869ef, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7bceb408522c41e0af49f97bcc429e12] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=35.3 K 2024-11-17T21:25:12,327 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:12,327 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/A is initiating minor compaction (all files) 2024-11-17T21:25:12,327 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/A in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:12,327 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/85d8a1ff24364e349e69888eca1e4387, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/9a9d9763d7324207ae25d14b566c838a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8566340f7900401e9038a52f67cb2761] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=35.3 K 2024-11-17T21:25:12,328 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85d8a1ff24364e349e69888eca1e4387, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878708196 2024-11-17T21:25:12,328 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 809fcb2b54c84a4d92a6657da549831c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878708196 2024-11-17T21:25:12,328 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a9d9763d7324207ae25d14b566c838a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731878708861 2024-11-17T21:25:12,328 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e0e33ac0ddb4553b6da997ae28869ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731878708861 2024-11-17T21:25:12,329 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8566340f7900401e9038a52f67cb2761, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731878710029 2024-11-17T21:25:12,329 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bceb408522c41e0af49f97bcc429e12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731878710029 2024-11-17T21:25:12,341 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#A#compaction#224 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:12,341 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#B#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:12,341 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/425e319ff944472cbce2af9dee3df1c8 is 50, key is test_row_0/A:col10/1731878711041/Put/seqid=0 2024-11-17T21:25:12,341 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/bb5db17cbea948a1971c3e9652378927 is 50, key is test_row_0/B:col10/1731878711041/Put/seqid=0 2024-11-17T21:25:12,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742094_1270 (size=12207) 2024-11-17T21:25:12,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742093_1269 (size=12207) 2024-11-17T21:25:12,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T21:25:12,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:12,429 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T21:25:12,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/2a1bb660584f446e81fae2c5cf0c5126 is 50, key is test_row_0/A:col10/1731878711113/Put/seqid=0 2024-11-17T21:25:12,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742095_1271 (size=12001) 2024-11-17T21:25:12,450 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/2a1bb660584f446e81fae2c5cf0c5126 2024-11-17T21:25:12,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/ffbd350f0f7e49f98e050b72217f7f69 is 50, key is test_row_0/B:col10/1731878711113/Put/seqid=0 2024-11-17T21:25:12,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742096_1272 (size=12001) 2024-11-17T21:25:12,780 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/bb5db17cbea948a1971c3e9652378927 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/bb5db17cbea948a1971c3e9652378927 2024-11-17T21:25:12,788 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 224e951b47a00740dfea2342be622072/B of 224e951b47a00740dfea2342be622072 into bb5db17cbea948a1971c3e9652378927(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:12,788 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:12,788 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/B, priority=13, startTime=1731878712325; duration=0sec 2024-11-17T21:25:12,788 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:12,788 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:B 2024-11-17T21:25:12,788 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:12,790 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:12,790 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/C is initiating minor compaction (all files) 2024-11-17T21:25:12,790 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/C in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:12,790 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/7f441302a8eb4edc9a1594c88c3f4990, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/b445dc700b844eaab91f7559a66ebfa9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/504166cda3ab4106bde8ce646679b909] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=35.3 K 2024-11-17T21:25:12,790 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f441302a8eb4edc9a1594c88c3f4990, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878708196 2024-11-17T21:25:12,792 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting b445dc700b844eaab91f7559a66ebfa9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731878708861 2024-11-17T21:25:12,792 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 504166cda3ab4106bde8ce646679b909, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731878710029 2024-11-17T21:25:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T21:25:12,803 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/425e319ff944472cbce2af9dee3df1c8 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/425e319ff944472cbce2af9dee3df1c8 2024-11-17T21:25:12,812 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 224e951b47a00740dfea2342be622072/A of 224e951b47a00740dfea2342be622072 into 425e319ff944472cbce2af9dee3df1c8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:12,812 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:12,812 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/A, priority=13, startTime=1731878712325; duration=0sec 2024-11-17T21:25:12,812 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:12,812 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:A 2024-11-17T21:25:12,817 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#C#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:12,818 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/3713dbc10f874079a0beda02fa62960c is 50, key is test_row_0/C:col10/1731878711041/Put/seqid=0 2024-11-17T21:25:12,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742097_1273 (size=12207) 2024-11-17T21:25:12,834 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/3713dbc10f874079a0beda02fa62960c as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/3713dbc10f874079a0beda02fa62960c 2024-11-17T21:25:12,843 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 224e951b47a00740dfea2342be622072/C of 224e951b47a00740dfea2342be622072 into 3713dbc10f874079a0beda02fa62960c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:12,843 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:12,843 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/C, priority=13, startTime=1731878712325; duration=0sec 2024-11-17T21:25:12,843 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:12,843 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:C 2024-11-17T21:25:12,867 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/ffbd350f0f7e49f98e050b72217f7f69 2024-11-17T21:25:12,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/6e27325accc749d4aad9d6f129cabd3e is 50, key is test_row_0/C:col10/1731878711113/Put/seqid=0 2024-11-17T21:25:12,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742098_1274 (size=12001) 2024-11-17T21:25:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:13,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:13,306 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/6e27325accc749d4aad9d6f129cabd3e 2024-11-17T21:25:13,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/2a1bb660584f446e81fae2c5cf0c5126 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/2a1bb660584f446e81fae2c5cf0c5126 2024-11-17T21:25:13,317 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/2a1bb660584f446e81fae2c5cf0c5126, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T21:25:13,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/ffbd350f0f7e49f98e050b72217f7f69 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/ffbd350f0f7e49f98e050b72217f7f69 2024-11-17T21:25:13,324 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/ffbd350f0f7e49f98e050b72217f7f69, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T21:25:13,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/6e27325accc749d4aad9d6f129cabd3e as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/6e27325accc749d4aad9d6f129cabd3e 2024-11-17T21:25:13,334 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/6e27325accc749d4aad9d6f129cabd3e, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T21:25:13,336 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=60.38 KB/61830 for 224e951b47a00740dfea2342be622072 in 906ms, sequenceid=114, compaction requested=false 2024-11-17T21:25:13,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:13,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:13,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-17T21:25:13,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-17T21:25:13,338 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:13,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T21:25:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:13,340 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-17T21:25:13,340 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6720 sec 2024-11-17T21:25:13,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.6790 sec 2024-11-17T21:25:13,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/106abcb1128b41f18a79c35e34803eaf is 50, key is test_row_0/A:col10/1731878713328/Put/seqid=0 2024-11-17T21:25:13,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742099_1275 (size=14391) 2024-11-17T21:25:13,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/106abcb1128b41f18a79c35e34803eaf 2024-11-17T21:25:13,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/49b9d0a3867143ceaa82de4b73f3b329 is 50, key is test_row_0/B:col10/1731878713328/Put/seqid=0 2024-11-17T21:25:13,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742100_1276 (size=12051) 2024-11-17T21:25:13,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/49b9d0a3867143ceaa82de4b73f3b329 2024-11-17T21:25:13,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/f68672f3800b400f84edecb1672ec952 is 50, key is test_row_0/C:col10/1731878713328/Put/seqid=0 2024-11-17T21:25:13,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742101_1277 (size=12051) 2024-11-17T21:25:13,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/f68672f3800b400f84edecb1672ec952 2024-11-17T21:25:13,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/106abcb1128b41f18a79c35e34803eaf as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/106abcb1128b41f18a79c35e34803eaf 2024-11-17T21:25:13,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878773442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:13,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/106abcb1128b41f18a79c35e34803eaf, entries=200, sequenceid=130, filesize=14.1 K 2024-11-17T21:25:13,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/49b9d0a3867143ceaa82de4b73f3b329 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/49b9d0a3867143ceaa82de4b73f3b329 2024-11-17T21:25:13,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/49b9d0a3867143ceaa82de4b73f3b329, entries=150, sequenceid=130, filesize=11.8 K 2024-11-17T21:25:13,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/f68672f3800b400f84edecb1672ec952 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/f68672f3800b400f84edecb1672ec952 2024-11-17T21:25:13,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/f68672f3800b400f84edecb1672ec952, entries=150, sequenceid=130, filesize=11.8 K 2024-11-17T21:25:13,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 224e951b47a00740dfea2342be622072 in 125ms, sequenceid=130, compaction requested=true 2024-11-17T21:25:13,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:13,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:25:13,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:13,463 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:13,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:13,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:13,463 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:13,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:25:13,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:13,465 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:13,465 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/B is initiating minor compaction (all files) 2024-11-17T21:25:13,465 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/B in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:13,465 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/bb5db17cbea948a1971c3e9652378927, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/ffbd350f0f7e49f98e050b72217f7f69, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/49b9d0a3867143ceaa82de4b73f3b329] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=35.4 K 2024-11-17T21:25:13,465 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:13,466 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/A is initiating minor compaction (all files) 2024-11-17T21:25:13,466 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/A in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:13,466 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/425e319ff944472cbce2af9dee3df1c8, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/2a1bb660584f446e81fae2c5cf0c5126, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/106abcb1128b41f18a79c35e34803eaf] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=37.7 K 2024-11-17T21:25:13,466 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting bb5db17cbea948a1971c3e9652378927, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731878710029 2024-11-17T21:25:13,466 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 425e319ff944472cbce2af9dee3df1c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731878710029 2024-11-17T21:25:13,467 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting ffbd350f0f7e49f98e050b72217f7f69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731878711068 2024-11-17T21:25:13,467 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a1bb660584f446e81fae2c5cf0c5126, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731878711068 2024-11-17T21:25:13,467 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 106abcb1128b41f18a79c35e34803eaf, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731878713291 2024-11-17T21:25:13,468 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 49b9d0a3867143ceaa82de4b73f3b329, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731878713291 2024-11-17T21:25:13,482 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#A#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:13,483 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/8e2583c13267475daf5f9c38d5b13c79 is 50, key is test_row_0/A:col10/1731878713328/Put/seqid=0 2024-11-17T21:25:13,486 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#B#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:13,486 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/d75a438522a14e22a2701c4b581a1027 is 50, key is test_row_0/B:col10/1731878713328/Put/seqid=0 2024-11-17T21:25:13,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742102_1278 (size=12359) 2024-11-17T21:25:13,551 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/d75a438522a14e22a2701c4b581a1027 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/d75a438522a14e22a2701c4b581a1027 2024-11-17T21:25:13,557 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:13,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T21:25:13,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742103_1279 (size=12359) 2024-11-17T21:25:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:13,562 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 224e951b47a00740dfea2342be622072/B of 224e951b47a00740dfea2342be622072 into d75a438522a14e22a2701c4b581a1027(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:13,562 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:13,562 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/B, priority=13, startTime=1731878713463; duration=0sec 2024-11-17T21:25:13,563 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:13,563 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:B 2024-11-17T21:25:13,563 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:13,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/30156ee08fd5424ca7ee51a5954d9e06 is 50, key is test_row_0/A:col10/1731878713440/Put/seqid=0 2024-11-17T21:25:13,567 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:13,567 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/C is initiating minor compaction (all files) 2024-11-17T21:25:13,567 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/C in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:13,567 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/3713dbc10f874079a0beda02fa62960c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/6e27325accc749d4aad9d6f129cabd3e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/f68672f3800b400f84edecb1672ec952] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=35.4 K 2024-11-17T21:25:13,568 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 3713dbc10f874079a0beda02fa62960c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731878710029 2024-11-17T21:25:13,568 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e27325accc749d4aad9d6f129cabd3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731878711068 2024-11-17T21:25:13,569 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting f68672f3800b400f84edecb1672ec952, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731878713291 2024-11-17T21:25:13,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742104_1280 (size=14541) 2024-11-17T21:25:13,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/30156ee08fd5424ca7ee51a5954d9e06 2024-11-17T21:25:13,585 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#C#compaction#236 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:13,586 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/8c98eaa8acd74714a6b5ae54ce5c6ef7 is 50, key is test_row_0/C:col10/1731878713328/Put/seqid=0 2024-11-17T21:25:13,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/2b75881f41d647308204c0c3ea28c4be is 50, key is test_row_0/B:col10/1731878713440/Put/seqid=0 2024-11-17T21:25:13,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742105_1281 (size=12359) 2024-11-17T21:25:13,621 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/8c98eaa8acd74714a6b5ae54ce5c6ef7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/8c98eaa8acd74714a6b5ae54ce5c6ef7 2024-11-17T21:25:13,629 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 224e951b47a00740dfea2342be622072/C of 224e951b47a00740dfea2342be622072 into 8c98eaa8acd74714a6b5ae54ce5c6ef7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:13,629 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:13,629 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/C, priority=13, startTime=1731878713463; duration=0sec 2024-11-17T21:25:13,629 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:13,630 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:C 2024-11-17T21:25:13,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878773635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:13,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742106_1282 (size=12151) 2024-11-17T21:25:13,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/2b75881f41d647308204c0c3ea28c4be 2024-11-17T21:25:13,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/a855d573906b47ca91c3e0c159f5263d is 50, key is test_row_0/C:col10/1731878713440/Put/seqid=0 2024-11-17T21:25:13,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742107_1283 (size=12151) 2024-11-17T21:25:13,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/a855d573906b47ca91c3e0c159f5263d 2024-11-17T21:25:13,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/30156ee08fd5424ca7ee51a5954d9e06 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/30156ee08fd5424ca7ee51a5954d9e06 2024-11-17T21:25:13,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/30156ee08fd5424ca7ee51a5954d9e06, entries=200, sequenceid=154, filesize=14.2 K 2024-11-17T21:25:13,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/2b75881f41d647308204c0c3ea28c4be as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/2b75881f41d647308204c0c3ea28c4be 2024-11-17T21:25:13,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/2b75881f41d647308204c0c3ea28c4be, entries=150, sequenceid=154, filesize=11.9 K 2024-11-17T21:25:13,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/a855d573906b47ca91c3e0c159f5263d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/a855d573906b47ca91c3e0c159f5263d 2024-11-17T21:25:13,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/a855d573906b47ca91c3e0c159f5263d, entries=150, sequenceid=154, filesize=11.9 K 2024-11-17T21:25:13,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 224e951b47a00740dfea2342be622072 in 154ms, sequenceid=154, compaction requested=false 2024-11-17T21:25:13,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:13,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:13,754 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:13,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T21:25:13,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/78b0c61a48404f5b9f2d3437390befb5 is 50, key is test_row_0/A:col10/1731878713634/Put/seqid=0 2024-11-17T21:25:13,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742108_1284 (size=14541) 2024-11-17T21:25:13,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T21:25:13,811 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:13,811 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:25:13,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-17T21:25:13,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T21:25:13,813 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:25:13,814 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:25:13,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:25:13,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878773864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:13,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T21:25:13,966 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/8e2583c13267475daf5f9c38d5b13c79 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8e2583c13267475daf5f9c38d5b13c79 2024-11-17T21:25:13,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T21:25:13,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:13,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:13,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:13,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:13,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:13,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878773971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:13,977 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 224e951b47a00740dfea2342be622072/A of 224e951b47a00740dfea2342be622072 into 8e2583c13267475daf5f9c38d5b13c79(size=12.1 K), total size for store is 26.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:13,977 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:13,977 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/A, priority=13, startTime=1731878713463; duration=0sec 2024-11-17T21:25:13,978 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:13,978 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:A 2024-11-17T21:25:14,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T21:25:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:14,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T21:25:14,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/78b0c61a48404f5b9f2d3437390befb5 2024-11-17T21:25:14,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878774181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:14,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/6e1ae008d6914dfdb5e3bcbf786848aa is 50, key is test_row_0/B:col10/1731878713634/Put/seqid=0 2024-11-17T21:25:14,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742109_1285 (size=12151) 2024-11-17T21:25:14,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878774201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:14,202 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_2' in 224e951b47a00740dfea2342be622072 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:14,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878774211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:14,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878774211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:14,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878774212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:14,215 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in 224e951b47a00740dfea2342be622072 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:14,215 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_2' in 224e951b47a00740dfea2342be622072 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:14,216 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in 224e951b47a00740dfea2342be622072 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:14,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T21:25:14,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:14,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:14,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:14,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T21:25:14,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:14,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:14,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:14,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T21:25:14,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878774491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:14,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:14,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:14,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:14,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T21:25:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:14,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/6e1ae008d6914dfdb5e3bcbf786848aa 2024-11-17T21:25:14,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/68843b5cc5b645c7809322fb3b3ad315 is 50, key is test_row_0/C:col10/1731878713634/Put/seqid=0 2024-11-17T21:25:14,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742110_1286 (size=12151) 2024-11-17T21:25:14,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T21:25:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:14,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T21:25:14,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:14,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:14,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:14,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T21:25:15,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878775001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:15,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/68843b5cc5b645c7809322fb3b3ad315 2024-11-17T21:25:15,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/78b0c61a48404f5b9f2d3437390befb5 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/78b0c61a48404f5b9f2d3437390befb5 2024-11-17T21:25:15,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/78b0c61a48404f5b9f2d3437390befb5, entries=200, sequenceid=169, filesize=14.2 K 2024-11-17T21:25:15,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/6e1ae008d6914dfdb5e3bcbf786848aa as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/6e1ae008d6914dfdb5e3bcbf786848aa 2024-11-17T21:25:15,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/6e1ae008d6914dfdb5e3bcbf786848aa, entries=150, sequenceid=169, filesize=11.9 K 2024-11-17T21:25:15,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/68843b5cc5b645c7809322fb3b3ad315 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/68843b5cc5b645c7809322fb3b3ad315 2024-11-17T21:25:15,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/68843b5cc5b645c7809322fb3b3ad315, entries=150, sequenceid=169, filesize=11.9 K 2024-11-17T21:25:15,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 224e951b47a00740dfea2342be622072 in 1284ms, sequenceid=169, compaction requested=true 2024-11-17T21:25:15,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:15,038 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:15,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:25:15,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:15,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:15,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:15,039 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:15,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:25:15,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:15,041 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:15,041 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/A is initiating minor compaction (all files) 2024-11-17T21:25:15,041 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:15,041 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/B is initiating minor compaction (all files) 2024-11-17T21:25:15,041 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/A in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:15,041 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/B in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:15,041 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/d75a438522a14e22a2701c4b581a1027, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/2b75881f41d647308204c0c3ea28c4be, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/6e1ae008d6914dfdb5e3bcbf786848aa] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=35.8 K 2024-11-17T21:25:15,041 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8e2583c13267475daf5f9c38d5b13c79, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/30156ee08fd5424ca7ee51a5954d9e06, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/78b0c61a48404f5b9f2d3437390befb5] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=40.5 K 2024-11-17T21:25:15,042 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d75a438522a14e22a2701c4b581a1027, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731878713291 2024-11-17T21:25:15,042 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e2583c13267475daf5f9c38d5b13c79, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731878713291 2024-11-17T21:25:15,043 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30156ee08fd5424ca7ee51a5954d9e06, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731878713404 2024-11-17T21:25:15,043 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b75881f41d647308204c0c3ea28c4be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731878713404 2024-11-17T21:25:15,043 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e1ae008d6914dfdb5e3bcbf786848aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731878713621 2024-11-17T21:25:15,043 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78b0c61a48404f5b9f2d3437390befb5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731878713610 2024-11-17T21:25:15,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T21:25:15,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:15,049 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T21:25:15,064 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#A#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:15,065 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/f17bc6b113f845ddba68a0d0e8665fdc is 50, key is test_row_0/A:col10/1731878713634/Put/seqid=0 2024-11-17T21:25:15,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/7d7980c5bd3d49a0b064ca1f5a4cb5b7 is 50, key is test_row_0/A:col10/1731878713856/Put/seqid=0 2024-11-17T21:25:15,078 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:15,079 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/e94fe94c4eab42529ec50ff994d1264b is 50, key is test_row_0/B:col10/1731878713634/Put/seqid=0 2024-11-17T21:25:15,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742111_1287 (size=12561) 2024-11-17T21:25:15,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742112_1288 (size=12151) 2024-11-17T21:25:15,100 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/f17bc6b113f845ddba68a0d0e8665fdc as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/f17bc6b113f845ddba68a0d0e8665fdc 2024-11-17T21:25:15,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742113_1289 (size=12561) 2024-11-17T21:25:15,106 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 224e951b47a00740dfea2342be622072/A of 224e951b47a00740dfea2342be622072 into f17bc6b113f845ddba68a0d0e8665fdc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:15,106 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:15,106 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/A, priority=13, startTime=1731878715038; duration=0sec 2024-11-17T21:25:15,106 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:15,106 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:A 2024-11-17T21:25:15,106 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:15,107 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:15,107 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/C is initiating minor compaction (all files) 2024-11-17T21:25:15,107 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/C in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:15,107 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/8c98eaa8acd74714a6b5ae54ce5c6ef7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/a855d573906b47ca91c3e0c159f5263d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/68843b5cc5b645c7809322fb3b3ad315] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=35.8 K 2024-11-17T21:25:15,108 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c98eaa8acd74714a6b5ae54ce5c6ef7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731878713291 2024-11-17T21:25:15,108 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting a855d573906b47ca91c3e0c159f5263d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731878713404 2024-11-17T21:25:15,108 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68843b5cc5b645c7809322fb3b3ad315, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731878713621 2024-11-17T21:25:15,114 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#C#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:15,115 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/9caefd24d06840c1ae63825738f45bbe is 50, key is test_row_0/C:col10/1731878713634/Put/seqid=0 2024-11-17T21:25:15,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742114_1290 (size=12561) 2024-11-17T21:25:15,133 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/9caefd24d06840c1ae63825738f45bbe as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/9caefd24d06840c1ae63825738f45bbe 2024-11-17T21:25:15,140 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 224e951b47a00740dfea2342be622072/C of 224e951b47a00740dfea2342be622072 into 9caefd24d06840c1ae63825738f45bbe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:15,140 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:15,140 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/C, priority=13, startTime=1731878715039; duration=0sec 2024-11-17T21:25:15,140 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:15,140 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:C 2024-11-17T21:25:15,499 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/7d7980c5bd3d49a0b064ca1f5a4cb5b7 2024-11-17T21:25:15,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/9572bb03e0344ef28affbbfcc504fa45 is 50, key is test_row_0/B:col10/1731878713856/Put/seqid=0 2024-11-17T21:25:15,511 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/e94fe94c4eab42529ec50ff994d1264b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e94fe94c4eab42529ec50ff994d1264b 2024-11-17T21:25:15,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742115_1291 (size=12151) 2024-11-17T21:25:15,521 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 224e951b47a00740dfea2342be622072/B of 224e951b47a00740dfea2342be622072 into e94fe94c4eab42529ec50ff994d1264b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:15,521 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:15,521 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/B, priority=13, startTime=1731878715039; duration=0sec 2024-11-17T21:25:15,521 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:15,521 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:B 2024-11-17T21:25:15,916 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/9572bb03e0344ef28affbbfcc504fa45 2024-11-17T21:25:15,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/76a564915b0b4a1e91dc3063a5b71965 is 50, key is test_row_0/C:col10/1731878713856/Put/seqid=0 2024-11-17T21:25:15,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742116_1292 (size=12151) 2024-11-17T21:25:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T21:25:16,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:16,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:16,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878776049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:16,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878776161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:16,350 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/76a564915b0b4a1e91dc3063a5b71965 2024-11-17T21:25:16,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/7d7980c5bd3d49a0b064ca1f5a4cb5b7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/7d7980c5bd3d49a0b064ca1f5a4cb5b7 2024-11-17T21:25:16,361 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/7d7980c5bd3d49a0b064ca1f5a4cb5b7, entries=150, sequenceid=193, filesize=11.9 K 2024-11-17T21:25:16,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/9572bb03e0344ef28affbbfcc504fa45 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/9572bb03e0344ef28affbbfcc504fa45 2024-11-17T21:25:16,367 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/9572bb03e0344ef28affbbfcc504fa45, entries=150, sequenceid=193, filesize=11.9 K 2024-11-17T21:25:16,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/76a564915b0b4a1e91dc3063a5b71965 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/76a564915b0b4a1e91dc3063a5b71965 2024-11-17T21:25:16,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878776371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:16,374 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/76a564915b0b4a1e91dc3063a5b71965, entries=150, sequenceid=193, filesize=11.9 K 2024-11-17T21:25:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:16,374 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 224e951b47a00740dfea2342be622072 in 1325ms, sequenceid=193, compaction requested=false 2024-11-17T21:25:16,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:16,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:16,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-17T21:25:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-17T21:25:16,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-17T21:25:16,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5630 sec 2024-11-17T21:25:16,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.5680 sec 2024-11-17T21:25:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:16,682 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:16,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T21:25:16,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/f6106569f5b4451bace82aa26ae02c79 is 50, key is test_row_0/A:col10/1731878716682/Put/seqid=0 2024-11-17T21:25:16,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742117_1293 (size=14541) 2024-11-17T21:25:16,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878776761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:16,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878776871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:16,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:16,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:16,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:17,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878777081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:17,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:17,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:17,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:17,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/f6106569f5b4451bace82aa26ae02c79 2024-11-17T21:25:17,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/e0e0df7f59ba451382c48ed8e00bb6cb is 50, key is test_row_0/B:col10/1731878716682/Put/seqid=0 2024-11-17T21:25:17,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742118_1294 (size=12151) 2024-11-17T21:25:17,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878777391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:17,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/e0e0df7f59ba451382c48ed8e00bb6cb 2024-11-17T21:25:17,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/3bb6d4150e0744f28a3336b7a73cc066 is 50, key is test_row_0/C:col10/1731878716682/Put/seqid=0 2024-11-17T21:25:17,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742119_1295 (size=12151) 2024-11-17T21:25:17,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/3bb6d4150e0744f28a3336b7a73cc066 2024-11-17T21:25:17,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/f6106569f5b4451bace82aa26ae02c79 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/f6106569f5b4451bace82aa26ae02c79 2024-11-17T21:25:17,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/f6106569f5b4451bace82aa26ae02c79, entries=200, sequenceid=210, filesize=14.2 K 2024-11-17T21:25:17,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/e0e0df7f59ba451382c48ed8e00bb6cb as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e0e0df7f59ba451382c48ed8e00bb6cb 2024-11-17T21:25:17,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e0e0df7f59ba451382c48ed8e00bb6cb, entries=150, sequenceid=210, filesize=11.9 K 2024-11-17T21:25:17,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/3bb6d4150e0744f28a3336b7a73cc066 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/3bb6d4150e0744f28a3336b7a73cc066 2024-11-17T21:25:17,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/3bb6d4150e0744f28a3336b7a73cc066, entries=150, sequenceid=210, filesize=11.9 K 2024-11-17T21:25:17,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 224e951b47a00740dfea2342be622072 in 864ms, sequenceid=210, compaction requested=true 2024-11-17T21:25:17,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:17,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:25:17,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:17,546 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:17,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:17,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:17,546 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:17,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:25:17,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:17,547 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:17,547 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/B is initiating minor compaction (all files) 2024-11-17T21:25:17,547 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:17,547 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/B in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:17,547 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/A is initiating minor compaction (all files) 2024-11-17T21:25:17,547 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/A in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:17,547 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e94fe94c4eab42529ec50ff994d1264b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/9572bb03e0344ef28affbbfcc504fa45, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e0e0df7f59ba451382c48ed8e00bb6cb] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=36.0 K 2024-11-17T21:25:17,547 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/f17bc6b113f845ddba68a0d0e8665fdc, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/7d7980c5bd3d49a0b064ca1f5a4cb5b7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/f6106569f5b4451bace82aa26ae02c79] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=38.3 K 2024-11-17T21:25:17,548 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting f17bc6b113f845ddba68a0d0e8665fdc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731878713621 2024-11-17T21:25:17,548 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting e94fe94c4eab42529ec50ff994d1264b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731878713621 2024-11-17T21:25:17,549 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d7980c5bd3d49a0b064ca1f5a4cb5b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731878713850 2024-11-17T21:25:17,549 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 9572bb03e0344ef28affbbfcc504fa45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731878713850 2024-11-17T21:25:17,549 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6106569f5b4451bace82aa26ae02c79, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731878716034 2024-11-17T21:25:17,549 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting e0e0df7f59ba451382c48ed8e00bb6cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731878716034 2024-11-17T21:25:17,555 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#B#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:17,555 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#A#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:17,555 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/54cb5dd6fda34ad780b7226d67ea49bd is 50, key is test_row_0/A:col10/1731878716682/Put/seqid=0 2024-11-17T21:25:17,555 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/27a6de79387d4dd1b33328fd2ccd6a09 is 50, key is test_row_0/B:col10/1731878716682/Put/seqid=0 2024-11-17T21:25:17,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742120_1296 (size=12663) 2024-11-17T21:25:17,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742121_1297 (size=12663) 2024-11-17T21:25:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:17,903 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:17,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T21:25:17,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/8a6480dc3024479db1287ca273f015bb is 50, key is test_row_0/A:col10/1731878716760/Put/seqid=0 2024-11-17T21:25:17,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742122_1298 (size=14541) 2024-11-17T21:25:17,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878777949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T21:25:17,971 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:17,971 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:25:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-17T21:25:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T21:25:17,973 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:25:17,973 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:25:17,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:25:17,982 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/27a6de79387d4dd1b33328fd2ccd6a09 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/27a6de79387d4dd1b33328fd2ccd6a09 2024-11-17T21:25:17,986 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 224e951b47a00740dfea2342be622072/B of 224e951b47a00740dfea2342be622072 into 27a6de79387d4dd1b33328fd2ccd6a09(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:17,986 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:17,986 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/B, priority=13, startTime=1731878717546; duration=0sec 2024-11-17T21:25:17,986 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:17,986 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:B 2024-11-17T21:25:17,986 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:17,987 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:17,987 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/C is initiating minor compaction (all files) 2024-11-17T21:25:17,987 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/C in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:17,987 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/9caefd24d06840c1ae63825738f45bbe, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/76a564915b0b4a1e91dc3063a5b71965, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/3bb6d4150e0744f28a3336b7a73cc066] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=36.0 K 2024-11-17T21:25:17,988 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 9caefd24d06840c1ae63825738f45bbe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731878713621 2024-11-17T21:25:17,988 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 76a564915b0b4a1e91dc3063a5b71965, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731878713850 2024-11-17T21:25:17,988 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bb6d4150e0744f28a3336b7a73cc066, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731878716034 2024-11-17T21:25:17,990 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/54cb5dd6fda34ad780b7226d67ea49bd as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/54cb5dd6fda34ad780b7226d67ea49bd 2024-11-17T21:25:17,995 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 224e951b47a00740dfea2342be622072/A of 224e951b47a00740dfea2342be622072 into 54cb5dd6fda34ad780b7226d67ea49bd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:17,995 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#C#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:17,995 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:17,995 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/A, priority=13, startTime=1731878717546; duration=0sec 2024-11-17T21:25:17,995 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:17,995 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:A 2024-11-17T21:25:17,996 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/7b0122af59724b32a5f983b1b2030b44 is 50, key is test_row_0/C:col10/1731878716682/Put/seqid=0 2024-11-17T21:25:17,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742123_1299 (size=12663) 2024-11-17T21:25:18,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878778061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:18,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T21:25:18,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T21:25:18,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:18,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:18,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:18,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36842 deadline: 1731878778231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:18,233 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_2' in 224e951b47a00740dfea2342be622072 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:18,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:18,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:18,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:18,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36900 deadline: 1731878778241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:18,242 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in 224e951b47a00740dfea2342be622072 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:18,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36840 deadline: 1731878778251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:18,253 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in 224e951b47a00740dfea2342be622072 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:18,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:18,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:18,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:18,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36834 deadline: 1731878778260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:18,263 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_2' in 224e951b47a00740dfea2342be622072 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:18,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:18,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:18,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:18,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878778270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:18,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:18,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:18,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:18,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T21:25:18,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:18,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:18,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:18,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T21:25:18,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/8a6480dc3024479db1287ca273f015bb 2024-11-17T21:25:18,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/c639997e3fb643dfbd988cbf190cafe6 is 50, key is test_row_0/B:col10/1731878716760/Put/seqid=0 2024-11-17T21:25:18,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742124_1300 (size=12151) 2024-11-17T21:25:18,403 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/7b0122af59724b32a5f983b1b2030b44 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/7b0122af59724b32a5f983b1b2030b44 2024-11-17T21:25:18,407 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 224e951b47a00740dfea2342be622072/C of 224e951b47a00740dfea2342be622072 into 7b0122af59724b32a5f983b1b2030b44(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:18,407 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:18,407 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/C, priority=13, startTime=1731878717546; duration=0sec 2024-11-17T21:25:18,407 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:18,407 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:C 2024-11-17T21:25:18,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T21:25:18,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:18,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:18,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:18,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T21:25:18,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:18,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:18,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:18,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878778581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T21:25:18,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/c639997e3fb643dfbd988cbf190cafe6 2024-11-17T21:25:18,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T21:25:18,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:18,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:18,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:18,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/f4f1b898bafe425f95d3ea233022dee7 is 50, key is test_row_0/C:col10/1731878716760/Put/seqid=0 2024-11-17T21:25:18,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742125_1301 (size=12151) 2024-11-17T21:25:18,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T21:25:18,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:18,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:18,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:18,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:19,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T21:25:19,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:19,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:19,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:19,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:19,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:19,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878779091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:19,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:19,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:19,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T21:25:19,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/f4f1b898bafe425f95d3ea233022dee7 2024-11-17T21:25:19,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/8a6480dc3024479db1287ca273f015bb as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8a6480dc3024479db1287ca273f015bb 2024-11-17T21:25:19,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8a6480dc3024479db1287ca273f015bb, entries=200, sequenceid=233, filesize=14.2 K 2024-11-17T21:25:19,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/c639997e3fb643dfbd988cbf190cafe6 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/c639997e3fb643dfbd988cbf190cafe6 2024-11-17T21:25:19,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/c639997e3fb643dfbd988cbf190cafe6, entries=150, sequenceid=233, filesize=11.9 K 2024-11-17T21:25:19,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/f4f1b898bafe425f95d3ea233022dee7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/f4f1b898bafe425f95d3ea233022dee7 2024-11-17T21:25:19,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/f4f1b898bafe425f95d3ea233022dee7, entries=150, sequenceid=233, filesize=11.9 K 2024-11-17T21:25:19,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 224e951b47a00740dfea2342be622072 in 1257ms, sequenceid=233, compaction requested=false 2024-11-17T21:25:19,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:19,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T21:25:19,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:19,193 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T21:25:19,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/725a6c9c525a4546a5e2d64d80b40610 is 50, key is test_row_0/A:col10/1731878717947/Put/seqid=0 2024-11-17T21:25:19,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742126_1302 (size=12151) 2024-11-17T21:25:19,601 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/725a6c9c525a4546a5e2d64d80b40610 2024-11-17T21:25:19,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/7779821f961d45a187aae388cb0b622c is 50, key is test_row_0/B:col10/1731878717947/Put/seqid=0 2024-11-17T21:25:19,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742127_1303 (size=12151) 2024-11-17T21:25:20,011 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/7779821f961d45a187aae388cb0b622c 2024-11-17T21:25:20,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/7fa62f223c1f477bb3508c12b71e1ef3 is 50, key is test_row_0/C:col10/1731878717947/Put/seqid=0 2024-11-17T21:25:20,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742128_1304 (size=12151) 2024-11-17T21:25:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:20,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:20,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T21:25:20,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878780187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:20,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:20,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:20,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:20,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878780300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:20,421 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/7fa62f223c1f477bb3508c12b71e1ef3 2024-11-17T21:25:20,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/725a6c9c525a4546a5e2d64d80b40610 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/725a6c9c525a4546a5e2d64d80b40610 2024-11-17T21:25:20,436 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/725a6c9c525a4546a5e2d64d80b40610, entries=150, sequenceid=249, filesize=11.9 K 2024-11-17T21:25:20,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/7779821f961d45a187aae388cb0b622c as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7779821f961d45a187aae388cb0b622c 2024-11-17T21:25:20,440 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7779821f961d45a187aae388cb0b622c, entries=150, sequenceid=249, filesize=11.9 K 2024-11-17T21:25:20,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/7fa62f223c1f477bb3508c12b71e1ef3 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/7fa62f223c1f477bb3508c12b71e1ef3 2024-11-17T21:25:20,444 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/7fa62f223c1f477bb3508c12b71e1ef3, entries=150, sequenceid=249, filesize=11.9 K 2024-11-17T21:25:20,445 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 224e951b47a00740dfea2342be622072 in 1252ms, sequenceid=249, compaction requested=true 2024-11-17T21:25:20,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:20,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:20,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-17T21:25:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-17T21:25:20,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-17T21:25:20,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4730 sec 2024-11-17T21:25:20,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.4780 sec 2024-11-17T21:25:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:20,513 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:20,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T21:25:20,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/b43c1405945a47b88639b16f9138d286 is 50, key is test_row_0/A:col10/1731878720154/Put/seqid=0 2024-11-17T21:25:20,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742129_1305 (size=14741) 2024-11-17T21:25:20,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878780545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:20,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878780651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:20,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878780861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:20,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:20,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:20,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:20,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/b43c1405945a47b88639b16f9138d286 2024-11-17T21:25:20,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/e2653700303c4b22883965f6f683fd2e is 50, key is test_row_0/B:col10/1731878720154/Put/seqid=0 2024-11-17T21:25:20,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742130_1306 (size=12301) 2024-11-17T21:25:21,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:21,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878781171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:21,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/e2653700303c4b22883965f6f683fd2e 2024-11-17T21:25:21,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/039bcaf9f52d475ab8378b99df18ba32 is 50, key is test_row_0/C:col10/1731878720154/Put/seqid=0 2024-11-17T21:25:21,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742131_1307 (size=12301) 2024-11-17T21:25:21,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/039bcaf9f52d475ab8378b99df18ba32 2024-11-17T21:25:21,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/b43c1405945a47b88639b16f9138d286 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/b43c1405945a47b88639b16f9138d286 2024-11-17T21:25:21,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/b43c1405945a47b88639b16f9138d286, entries=200, sequenceid=273, filesize=14.4 K 2024-11-17T21:25:21,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/e2653700303c4b22883965f6f683fd2e as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e2653700303c4b22883965f6f683fd2e 2024-11-17T21:25:21,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e2653700303c4b22883965f6f683fd2e, entries=150, sequenceid=273, filesize=12.0 K 2024-11-17T21:25:21,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/039bcaf9f52d475ab8378b99df18ba32 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/039bcaf9f52d475ab8378b99df18ba32 2024-11-17T21:25:21,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/039bcaf9f52d475ab8378b99df18ba32, entries=150, sequenceid=273, filesize=12.0 K 2024-11-17T21:25:21,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 224e951b47a00740dfea2342be622072 in 852ms, sequenceid=273, compaction requested=true 2024-11-17T21:25:21,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:21,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:25:21,365 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:21,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:21,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:21,366 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:21,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:21,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:25:21,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:21,367 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:21,367 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54096 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:21,367 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/B is initiating minor compaction (all files) 2024-11-17T21:25:21,367 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/A is initiating minor compaction (all files) 2024-11-17T21:25:21,367 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/A in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:21,367 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/B in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:21,367 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/54cb5dd6fda34ad780b7226d67ea49bd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8a6480dc3024479db1287ca273f015bb, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/725a6c9c525a4546a5e2d64d80b40610, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/b43c1405945a47b88639b16f9138d286] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=52.8 K 2024-11-17T21:25:21,367 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/27a6de79387d4dd1b33328fd2ccd6a09, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/c639997e3fb643dfbd988cbf190cafe6, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7779821f961d45a187aae388cb0b622c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e2653700303c4b22883965f6f683fd2e] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=48.1 K 2024-11-17T21:25:21,367 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54cb5dd6fda34ad780b7226d67ea49bd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731878716034 2024-11-17T21:25:21,367 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 27a6de79387d4dd1b33328fd2ccd6a09, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731878716034 2024-11-17T21:25:21,368 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting c639997e3fb643dfbd988cbf190cafe6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731878716752 2024-11-17T21:25:21,368 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a6480dc3024479db1287ca273f015bb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731878716752 2024-11-17T21:25:21,368 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 7779821f961d45a187aae388cb0b622c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731878717923 2024-11-17T21:25:21,369 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 725a6c9c525a4546a5e2d64d80b40610, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731878717923 2024-11-17T21:25:21,369 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting e2653700303c4b22883965f6f683fd2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731878720154 2024-11-17T21:25:21,369 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting b43c1405945a47b88639b16f9138d286, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731878720154 2024-11-17T21:25:21,380 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#B#compaction#263 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:21,382 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#A#compaction#264 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:21,382 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/7792742aaf614ca7ab37950d5561bb1d is 50, key is test_row_0/B:col10/1731878720154/Put/seqid=0 2024-11-17T21:25:21,382 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/12377c6730d644698cb64aa1930bcd13 is 50, key is test_row_0/A:col10/1731878720154/Put/seqid=0 2024-11-17T21:25:21,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742132_1308 (size=12949) 2024-11-17T21:25:21,408 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/7792742aaf614ca7ab37950d5561bb1d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7792742aaf614ca7ab37950d5561bb1d 2024-11-17T21:25:21,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742133_1309 (size=12949) 2024-11-17T21:25:21,414 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 224e951b47a00740dfea2342be622072/B of 224e951b47a00740dfea2342be622072 into 7792742aaf614ca7ab37950d5561bb1d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:21,414 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:21,414 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/B, priority=12, startTime=1731878721365; duration=0sec 2024-11-17T21:25:21,414 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:21,414 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:B 2024-11-17T21:25:21,414 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:21,415 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:21,416 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/C is initiating minor compaction (all files) 2024-11-17T21:25:21,416 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/C in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:21,416 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/7b0122af59724b32a5f983b1b2030b44, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/f4f1b898bafe425f95d3ea233022dee7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/7fa62f223c1f477bb3508c12b71e1ef3, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/039bcaf9f52d475ab8378b99df18ba32] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=48.1 K 2024-11-17T21:25:21,416 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b0122af59724b32a5f983b1b2030b44, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731878716034 2024-11-17T21:25:21,416 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting f4f1b898bafe425f95d3ea233022dee7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731878716752 2024-11-17T21:25:21,417 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fa62f223c1f477bb3508c12b71e1ef3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731878717923 2024-11-17T21:25:21,417 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 039bcaf9f52d475ab8378b99df18ba32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731878720154 2024-11-17T21:25:21,424 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#C#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:21,424 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/1cc1cfd7c6f949e1a5a29119ef650626 is 50, key is test_row_0/C:col10/1731878720154/Put/seqid=0 2024-11-17T21:25:21,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742134_1310 (size=12949) 2024-11-17T21:25:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:21,682 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:21,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T21:25:21,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/5ae8cb7093494298920f25d0292d6cfb is 50, key is test_row_0/A:col10/1731878720544/Put/seqid=0 2024-11-17T21:25:21,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742135_1311 (size=12301) 2024-11-17T21:25:21,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:21,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878781761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:21,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:21,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:21,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:21,813 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/12377c6730d644698cb64aa1930bcd13 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/12377c6730d644698cb64aa1930bcd13 2024-11-17T21:25:21,821 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 224e951b47a00740dfea2342be622072/A of 224e951b47a00740dfea2342be622072 into 12377c6730d644698cb64aa1930bcd13(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:21,821 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:21,821 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/A, priority=12, startTime=1731878721365; duration=0sec 2024-11-17T21:25:21,821 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:21,821 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:A 2024-11-17T21:25:21,833 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/1cc1cfd7c6f949e1a5a29119ef650626 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/1cc1cfd7c6f949e1a5a29119ef650626 2024-11-17T21:25:21,838 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 224e951b47a00740dfea2342be622072/C of 224e951b47a00740dfea2342be622072 into 1cc1cfd7c6f949e1a5a29119ef650626(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:21,838 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:21,838 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/C, priority=12, startTime=1731878721366; duration=0sec 2024-11-17T21:25:21,838 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:21,838 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:C 2024-11-17T21:25:21,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878781871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:22,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878782080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:22,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/5ae8cb7093494298920f25d0292d6cfb 2024-11-17T21:25:22,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/c9b8408c4bca4565b9b81aed9333522e is 50, key is test_row_0/B:col10/1731878720544/Put/seqid=0 2024-11-17T21:25:22,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742136_1312 (size=12301) 2024-11-17T21:25:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T21:25:22,151 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:22,152 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:25:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-17T21:25:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T21:25:22,154 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:25:22,154 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:25:22,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:25:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T21:25:22,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T21:25:22,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:22,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:22,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:22,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:22,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:22,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878782391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:22,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T21:25:22,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:22,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:22,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:22,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:22,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T21:25:22,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/c9b8408c4bca4565b9b81aed9333522e 2024-11-17T21:25:22,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/738b7d0f89f54409b92cc68cd45f403d is 50, key is test_row_0/C:col10/1731878720544/Put/seqid=0 2024-11-17T21:25:22,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742137_1313 (size=12301) 2024-11-17T21:25:22,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T21:25:22,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:22,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:22,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:22,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:22,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:22,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T21:25:22,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:22,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:22,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:22,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:22,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T21:25:22,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878782900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:22,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:22,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:22,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:22,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T21:25:22,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:22,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:22,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:22,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:22,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:22,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/738b7d0f89f54409b92cc68cd45f403d 2024-11-17T21:25:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:22,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/5ae8cb7093494298920f25d0292d6cfb as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/5ae8cb7093494298920f25d0292d6cfb 2024-11-17T21:25:22,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/5ae8cb7093494298920f25d0292d6cfb, entries=150, sequenceid=287, filesize=12.0 K 2024-11-17T21:25:22,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/c9b8408c4bca4565b9b81aed9333522e as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/c9b8408c4bca4565b9b81aed9333522e 2024-11-17T21:25:22,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/c9b8408c4bca4565b9b81aed9333522e, entries=150, sequenceid=287, filesize=12.0 K 2024-11-17T21:25:22,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/738b7d0f89f54409b92cc68cd45f403d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/738b7d0f89f54409b92cc68cd45f403d 2024-11-17T21:25:22,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/738b7d0f89f54409b92cc68cd45f403d, entries=150, sequenceid=287, filesize=12.0 K 2024-11-17T21:25:22,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 224e951b47a00740dfea2342be622072 in 1266ms, sequenceid=287, compaction requested=false 2024-11-17T21:25:22,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:23,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T21:25:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:23,078 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T21:25:23,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/779fae0977944b058f9b8d95023d1a6a is 50, key is test_row_0/A:col10/1731878721760/Put/seqid=0 2024-11-17T21:25:23,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742138_1314 (size=12301) 2024-11-17T21:25:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T21:25:23,498 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/779fae0977944b058f9b8d95023d1a6a 2024-11-17T21:25:23,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/6731cad9e0d444d7a8606860b11afeca is 50, key is test_row_0/B:col10/1731878721760/Put/seqid=0 2024-11-17T21:25:23,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742139_1315 (size=12301) 2024-11-17T21:25:23,911 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/6731cad9e0d444d7a8606860b11afeca 2024-11-17T21:25:23,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:23,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/776ea5e6f737473aab965206222c7d82 is 50, key is test_row_0/C:col10/1731878721760/Put/seqid=0 2024-11-17T21:25:23,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742140_1316 (size=12301) 2024-11-17T21:25:23,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878783947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:24,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878784061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:24,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:24,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:24,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:24,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878784270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T21:25:24,333 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/776ea5e6f737473aab965206222c7d82 2024-11-17T21:25:24,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/779fae0977944b058f9b8d95023d1a6a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/779fae0977944b058f9b8d95023d1a6a 2024-11-17T21:25:24,343 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/779fae0977944b058f9b8d95023d1a6a, entries=150, sequenceid=312, filesize=12.0 K 2024-11-17T21:25:24,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/6731cad9e0d444d7a8606860b11afeca as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/6731cad9e0d444d7a8606860b11afeca 2024-11-17T21:25:24,348 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/6731cad9e0d444d7a8606860b11afeca, entries=150, sequenceid=312, filesize=12.0 K 2024-11-17T21:25:24,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/776ea5e6f737473aab965206222c7d82 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/776ea5e6f737473aab965206222c7d82 2024-11-17T21:25:24,355 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/776ea5e6f737473aab965206222c7d82, entries=150, sequenceid=312, filesize=12.0 K 2024-11-17T21:25:24,356 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 224e951b47a00740dfea2342be622072 in 1278ms, sequenceid=312, compaction requested=true 2024-11-17T21:25:24,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:24,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:24,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-17T21:25:24,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-17T21:25:24,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-17T21:25:24,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2030 sec 2024-11-17T21:25:24,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.2070 sec 2024-11-17T21:25:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:24,584 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:24,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T21:25:24,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/9440288688494312b2e51757f366bbb3 is 50, key is test_row_0/A:col10/1731878724583/Put/seqid=0 2024-11-17T21:25:24,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742141_1317 (size=14741) 2024-11-17T21:25:24,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878784662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:24,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878784770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:24,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:24,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:24,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:24,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878784981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:24,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/9440288688494312b2e51757f366bbb3 2024-11-17T21:25:24,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/1fa6a4d199454bb1ad8a69d4cc795e16 is 50, key is test_row_0/B:col10/1731878724583/Put/seqid=0 2024-11-17T21:25:25,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742142_1318 (size=12301) 2024-11-17T21:25:25,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878785291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:25,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/1fa6a4d199454bb1ad8a69d4cc795e16 2024-11-17T21:25:25,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/aaf557f433cc4464894dd3222b18fee1 is 50, key is test_row_0/C:col10/1731878724583/Put/seqid=0 2024-11-17T21:25:25,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742143_1319 (size=12301) 2024-11-17T21:25:25,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878785801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:25,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:25,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:25,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:25,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/aaf557f433cc4464894dd3222b18fee1 2024-11-17T21:25:25,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/9440288688494312b2e51757f366bbb3 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/9440288688494312b2e51757f366bbb3 2024-11-17T21:25:25,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/9440288688494312b2e51757f366bbb3, entries=200, sequenceid=326, filesize=14.4 K 2024-11-17T21:25:25,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/1fa6a4d199454bb1ad8a69d4cc795e16 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/1fa6a4d199454bb1ad8a69d4cc795e16 2024-11-17T21:25:25,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/1fa6a4d199454bb1ad8a69d4cc795e16, entries=150, sequenceid=326, filesize=12.0 K 2024-11-17T21:25:25,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/aaf557f433cc4464894dd3222b18fee1 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/aaf557f433cc4464894dd3222b18fee1 2024-11-17T21:25:25,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/aaf557f433cc4464894dd3222b18fee1, entries=150, sequenceid=326, filesize=12.0 K 2024-11-17T21:25:25,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 224e951b47a00740dfea2342be622072 in 1250ms, sequenceid=326, compaction requested=true 2024-11-17T21:25:25,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:25,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:25:25,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:25,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:25,835 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:25,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:25,835 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:25,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:25:25,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:25,836 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:25,836 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:25,836 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/B is initiating minor compaction (all files) 2024-11-17T21:25:25,836 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/A is initiating minor compaction (all files) 2024-11-17T21:25:25,836 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/A in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:25,836 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/B in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:25,836 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/12377c6730d644698cb64aa1930bcd13, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/5ae8cb7093494298920f25d0292d6cfb, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/779fae0977944b058f9b8d95023d1a6a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/9440288688494312b2e51757f366bbb3] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=51.1 K 2024-11-17T21:25:25,836 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7792742aaf614ca7ab37950d5561bb1d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/c9b8408c4bca4565b9b81aed9333522e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/6731cad9e0d444d7a8606860b11afeca, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/1fa6a4d199454bb1ad8a69d4cc795e16] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=48.7 K 2024-11-17T21:25:25,837 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12377c6730d644698cb64aa1930bcd13, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731878720154 2024-11-17T21:25:25,837 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 7792742aaf614ca7ab37950d5561bb1d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731878720154 2024-11-17T21:25:25,837 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting c9b8408c4bca4565b9b81aed9333522e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731878720536 2024-11-17T21:25:25,837 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ae8cb7093494298920f25d0292d6cfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731878720536 2024-11-17T21:25:25,837 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 779fae0977944b058f9b8d95023d1a6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731878721747 2024-11-17T21:25:25,837 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 6731cad9e0d444d7a8606860b11afeca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731878721747 2024-11-17T21:25:25,838 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9440288688494312b2e51757f366bbb3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731878723923 2024-11-17T21:25:25,838 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fa6a4d199454bb1ad8a69d4cc795e16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731878723923 2024-11-17T21:25:25,859 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#A#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:25,859 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/fb4d76c6441142d2a60f34ae6ae486a4 is 50, key is test_row_0/A:col10/1731878724583/Put/seqid=0 2024-11-17T21:25:25,859 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#B#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:25,860 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/38cf412b24404cba9db56b50984a5373 is 50, key is test_row_0/B:col10/1731878724583/Put/seqid=0 2024-11-17T21:25:25,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742145_1321 (size=13085) 2024-11-17T21:25:25,892 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/fb4d76c6441142d2a60f34ae6ae486a4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/fb4d76c6441142d2a60f34ae6ae486a4 2024-11-17T21:25:25,899 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 224e951b47a00740dfea2342be622072/A of 224e951b47a00740dfea2342be622072 into fb4d76c6441142d2a60f34ae6ae486a4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:25,899 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:25,899 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/A, priority=12, startTime=1731878725834; duration=0sec 2024-11-17T21:25:25,899 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:25,899 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:A 2024-11-17T21:25:25,900 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:25,901 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:25,901 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 224e951b47a00740dfea2342be622072/C is initiating minor compaction (all files) 2024-11-17T21:25:25,901 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 224e951b47a00740dfea2342be622072/C in TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:25,902 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/1cc1cfd7c6f949e1a5a29119ef650626, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/738b7d0f89f54409b92cc68cd45f403d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/776ea5e6f737473aab965206222c7d82, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/aaf557f433cc4464894dd3222b18fee1] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp, totalSize=48.7 K 2024-11-17T21:25:25,902 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cc1cfd7c6f949e1a5a29119ef650626, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731878720154 2024-11-17T21:25:25,902 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 738b7d0f89f54409b92cc68cd45f403d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731878720536 2024-11-17T21:25:25,902 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 776ea5e6f737473aab965206222c7d82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731878721747 2024-11-17T21:25:25,903 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaf557f433cc4464894dd3222b18fee1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731878723923 2024-11-17T21:25:25,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742144_1320 (size=13085) 2024-11-17T21:25:25,911 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 224e951b47a00740dfea2342be622072#C#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:25,911 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/38cf412b24404cba9db56b50984a5373 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/38cf412b24404cba9db56b50984a5373 2024-11-17T21:25:25,911 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/8e2f05e05289410591b25881d0af2086 is 50, key is test_row_0/C:col10/1731878724583/Put/seqid=0 2024-11-17T21:25:25,916 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 224e951b47a00740dfea2342be622072/B of 224e951b47a00740dfea2342be622072 into 38cf412b24404cba9db56b50984a5373(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:25,916 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:25,916 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/B, priority=12, startTime=1731878725834; duration=0sec 2024-11-17T21:25:25,917 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:25,917 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:B 2024-11-17T21:25:25,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742146_1322 (size=13085) 2024-11-17T21:25:25,927 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/8e2f05e05289410591b25881d0af2086 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/8e2f05e05289410591b25881d0af2086 2024-11-17T21:25:25,931 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 224e951b47a00740dfea2342be622072/C of 224e951b47a00740dfea2342be622072 into 8e2f05e05289410591b25881d0af2086(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:25,931 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:25,931 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., storeName=224e951b47a00740dfea2342be622072/C, priority=12, startTime=1731878725835; duration=0sec 2024-11-17T21:25:25,931 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:25,931 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:C 2024-11-17T21:25:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T21:25:26,331 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:26,331 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:25:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-17T21:25:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T21:25:26,333 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:25:26,334 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:25:26,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:25:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T21:25:26,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T21:25:26,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:26,486 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T21:25:26,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/7b4c3720b68d4409adf2674fc51fd381 is 50, key is test_row_0/A:col10/1731878724656/Put/seqid=0 2024-11-17T21:25:26,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742147_1323 (size=12301) 2024-11-17T21:25:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T21:25:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:26,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. as already flushing 2024-11-17T21:25:26,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878786843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:26,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:26,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:26,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:26,894 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/7b4c3720b68d4409adf2674fc51fd381 2024-11-17T21:25:26,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/bb35d7afb6034bf5b8c68fb977dffc13 is 50, key is test_row_0/B:col10/1731878724656/Put/seqid=0 2024-11-17T21:25:26,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742148_1324 (size=12301) 2024-11-17T21:25:26,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878786951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T21:25:27,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878787161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:27,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:27,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:27,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:27,303 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/bb35d7afb6034bf5b8c68fb977dffc13 2024-11-17T21:25:27,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/4a48bfec3cf14c77be12da1d141e5e37 is 50, key is test_row_0/C:col10/1731878724656/Put/seqid=0 2024-11-17T21:25:27,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742149_1325 (size=12301) 2024-11-17T21:25:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T21:25:27,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36896 deadline: 1731878787470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:27,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:27,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:27,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:25:27,524 INFO [Thread-1171 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1171. 2024-11-17T21:25:27,524 DEBUG [Thread-1171 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:27,524 DEBUG [Thread-1171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:27,524 DEBUG [Thread-1171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:27,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:27,524 INFO [Thread-1173 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1173. 2024-11-17T21:25:27,524 DEBUG [Thread-1173 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:27,524 DEBUG [Thread-1173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:27,524 DEBUG [Thread-1173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:27,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:27,525 INFO [Thread-1175 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1175. 2024-11-17T21:25:27,525 DEBUG [Thread-1175 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:27,525 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:27,525 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:27,525 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:27,526 INFO [Thread-1177 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1177. 2024-11-17T21:25:27,526 DEBUG [Thread-1177 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:27,527 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:27,527 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:27,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:27,527 INFO [Thread-1179 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1179. 2024-11-17T21:25:27,527 DEBUG [Thread-1179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:27,527 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:27,527 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:27,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:27,714 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/4a48bfec3cf14c77be12da1d141e5e37 2024-11-17T21:25:27,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/7b4c3720b68d4409adf2674fc51fd381 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/7b4c3720b68d4409adf2674fc51fd381 2024-11-17T21:25:27,725 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/7b4c3720b68d4409adf2674fc51fd381, entries=150, sequenceid=351, filesize=12.0 K 2024-11-17T21:25:27,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/bb35d7afb6034bf5b8c68fb977dffc13 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/bb35d7afb6034bf5b8c68fb977dffc13 2024-11-17T21:25:27,730 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/bb35d7afb6034bf5b8c68fb977dffc13, entries=150, sequenceid=351, filesize=12.0 K 2024-11-17T21:25:27,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/4a48bfec3cf14c77be12da1d141e5e37 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/4a48bfec3cf14c77be12da1d141e5e37 2024-11-17T21:25:27,735 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/4a48bfec3cf14c77be12da1d141e5e37, entries=150, sequenceid=351, filesize=12.0 K 2024-11-17T21:25:27,735 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 224e951b47a00740dfea2342be622072 in 1249ms, sequenceid=351, compaction requested=false 2024-11-17T21:25:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-17T21:25:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-17T21:25:27,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-17T21:25:27,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4030 sec 2024-11-17T21:25:27,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.4080 sec 2024-11-17T21:25:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 224e951b47a00740dfea2342be622072 2024-11-17T21:25:27,985 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:27,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T21:25:27,986 INFO [Thread-1164 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1164. 2024-11-17T21:25:27,986 DEBUG [Thread-1164 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:27,986 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:27,986 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:27,986 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:27,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/d529d618b7e34a1fbc5ec33ca8aae189 is 50, key is test_row_0/A:col10/1731878726834/Put/seqid=0 2024-11-17T21:25:27,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742150_1326 (size=12301) 2024-11-17T21:25:28,264 INFO [Thread-1168 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1168. 2024-11-17T21:25:28,264 INFO [Thread-1162 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1162. 2024-11-17T21:25:28,264 DEBUG [Thread-1168 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:28,264 DEBUG [Thread-1162 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:28,264 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:28,264 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:28,265 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:28,265 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:28,265 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:28,265 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:28,334 INFO [Thread-1160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1160. 2024-11-17T21:25:28,334 DEBUG [Thread-1160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:28,334 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:28,335 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:28,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:28,375 INFO [Thread-1166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1166. 2024-11-17T21:25:28,376 DEBUG [Thread-1166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:28,376 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:28,376 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:28,376 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:28,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/d529d618b7e34a1fbc5ec33ca8aae189 2024-11-17T21:25:28,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/dc3f2fe0de014f139a316cf29fd0c3ab is 50, key is test_row_0/B:col10/1731878726834/Put/seqid=0 2024-11-17T21:25:28,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742151_1327 (size=12301) 2024-11-17T21:25:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T21:25:28,482 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:28,482 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T21:25:28,482 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-17T21:25:28,482 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-17T21:25:28,482 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 230 2024-11-17T21:25:28,482 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-17T21:25:28,482 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-17T21:25:28,482 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T21:25:28,482 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4158 2024-11-17T21:25:28,482 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3982 2024-11-17T21:25:28,482 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4016 2024-11-17T21:25:28,482 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4162 2024-11-17T21:25:28,482 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3976 2024-11-17T21:25:28,482 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T21:25:28,482 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T21:25:28,482 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T21:25:28,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:28,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:28,483 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:28,484 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T21:25:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T21:25:28,491 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878728490"}]},"ts":"1731878728490"} 2024-11-17T21:25:28,493 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T21:25:28,493 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T21:25:28,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T21:25:28,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=224e951b47a00740dfea2342be622072, UNASSIGN}] 2024-11-17T21:25:28,497 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=224e951b47a00740dfea2342be622072, UNASSIGN 2024-11-17T21:25:28,497 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=224e951b47a00740dfea2342be622072, regionState=CLOSING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:25:28,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=224e951b47a00740dfea2342be622072, UNASSIGN because future has completed 2024-11-17T21:25:28,499 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T21:25:28,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:25:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T21:25:28,652 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 224e951b47a00740dfea2342be622072 2024-11-17T21:25:28,652 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T21:25:28,652 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 224e951b47a00740dfea2342be622072, disabling compactions & flushes 2024-11-17T21:25:28,652 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T21:25:28,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/dc3f2fe0de014f139a316cf29fd0c3ab 2024-11-17T21:25:28,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/84b6be5b2fa14199b2064703e675aad7 is 50, key is test_row_0/C:col10/1731878726834/Put/seqid=0 2024-11-17T21:25:28,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742152_1328 (size=12301) 2024-11-17T21:25:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T21:25:29,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/84b6be5b2fa14199b2064703e675aad7 2024-11-17T21:25:29,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/d529d618b7e34a1fbc5ec33ca8aae189 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/d529d618b7e34a1fbc5ec33ca8aae189 2024-11-17T21:25:29,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/d529d618b7e34a1fbc5ec33ca8aae189, entries=150, sequenceid=366, filesize=12.0 K 2024-11-17T21:25:29,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/dc3f2fe0de014f139a316cf29fd0c3ab as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/dc3f2fe0de014f139a316cf29fd0c3ab 2024-11-17T21:25:29,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/dc3f2fe0de014f139a316cf29fd0c3ab, entries=150, sequenceid=366, filesize=12.0 K 2024-11-17T21:25:29,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/84b6be5b2fa14199b2064703e675aad7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/84b6be5b2fa14199b2064703e675aad7 2024-11-17T21:25:29,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/84b6be5b2fa14199b2064703e675aad7, entries=150, sequenceid=366, filesize=12.0 K 2024-11-17T21:25:29,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=26.84 KB/27480 for 224e951b47a00740dfea2342be622072 in 1272ms, sequenceid=366, compaction requested=true 2024-11-17T21:25:29,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 224e951b47a00740dfea2342be622072: 2024-11-17T21:25:29,257 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:29,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:25:29,257 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:29,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:29,257 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. because compaction request was cancelled 2024-11-17T21:25:29,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:29,257 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. after waiting 0 ms 2024-11-17T21:25:29,257 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:A 2024-11-17T21:25:29,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:29,257 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. because compaction request was cancelled 2024-11-17T21:25:29,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 224e951b47a00740dfea2342be622072:C, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:29,257 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:29,257 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:B 2024-11-17T21:25:29,257 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. because compaction request was cancelled 2024-11-17T21:25:29,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:29,257 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 224e951b47a00740dfea2342be622072:C 2024-11-17T21:25:29,257 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing 224e951b47a00740dfea2342be622072 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-17T21:25:29,260 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/de110c4bde5b46469e5ca5867c727168 is 50, key is test_row_1/A:col10/1731878728262/Put/seqid=0 2024-11-17T21:25:29,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742153_1329 (size=9857) 2024-11-17T21:25:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T21:25:29,665 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/de110c4bde5b46469e5ca5867c727168 2024-11-17T21:25:29,679 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/18eadad8918b4cd8b133816465458d4d is 50, key is test_row_1/B:col10/1731878728262/Put/seqid=0 2024-11-17T21:25:29,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742154_1330 (size=9857) 2024-11-17T21:25:30,087 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/18eadad8918b4cd8b133816465458d4d 2024-11-17T21:25:30,100 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/228350aa200b468ab5d70d0ac708a740 is 50, key is test_row_1/C:col10/1731878728262/Put/seqid=0 2024-11-17T21:25:30,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742155_1331 (size=9857) 2024-11-17T21:25:30,505 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/228350aa200b468ab5d70d0ac708a740 2024-11-17T21:25:30,515 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/A/de110c4bde5b46469e5ca5867c727168 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/de110c4bde5b46469e5ca5867c727168 2024-11-17T21:25:30,522 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/de110c4bde5b46469e5ca5867c727168, entries=100, sequenceid=373, filesize=9.6 K 2024-11-17T21:25:30,524 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/B/18eadad8918b4cd8b133816465458d4d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/18eadad8918b4cd8b133816465458d4d 2024-11-17T21:25:30,528 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/18eadad8918b4cd8b133816465458d4d, entries=100, sequenceid=373, filesize=9.6 K 2024-11-17T21:25:30,529 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/.tmp/C/228350aa200b468ab5d70d0ac708a740 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/228350aa200b468ab5d70d0ac708a740 2024-11-17T21:25:30,532 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/228350aa200b468ab5d70d0ac708a740, entries=100, sequenceid=373, filesize=9.6 K 2024-11-17T21:25:30,533 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 224e951b47a00740dfea2342be622072 in 1276ms, sequenceid=373, compaction requested=true 2024-11-17T21:25:30,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/a6c4f7330a1f4f1a965ad8ec17ffdf77, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/cc092080ab514ca2a18c90a3eedcfabf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/85d8a1ff24364e349e69888eca1e4387, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/ebd124ff6fec4060890a81c2baac7d72, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/9a9d9763d7324207ae25d14b566c838a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/425e319ff944472cbce2af9dee3df1c8, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8566340f7900401e9038a52f67cb2761, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/2a1bb660584f446e81fae2c5cf0c5126, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/106abcb1128b41f18a79c35e34803eaf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8e2583c13267475daf5f9c38d5b13c79, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/30156ee08fd5424ca7ee51a5954d9e06, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/78b0c61a48404f5b9f2d3437390befb5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/f17bc6b113f845ddba68a0d0e8665fdc, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/7d7980c5bd3d49a0b064ca1f5a4cb5b7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/f6106569f5b4451bace82aa26ae02c79, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/54cb5dd6fda34ad780b7226d67ea49bd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8a6480dc3024479db1287ca273f015bb, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/725a6c9c525a4546a5e2d64d80b40610, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/b43c1405945a47b88639b16f9138d286, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/12377c6730d644698cb64aa1930bcd13, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/5ae8cb7093494298920f25d0292d6cfb, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/779fae0977944b058f9b8d95023d1a6a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/9440288688494312b2e51757f366bbb3] to archive 2024-11-17T21:25:30,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:25:30,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/a6c4f7330a1f4f1a965ad8ec17ffdf77 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/a6c4f7330a1f4f1a965ad8ec17ffdf77 2024-11-17T21:25:30,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/cc092080ab514ca2a18c90a3eedcfabf to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/cc092080ab514ca2a18c90a3eedcfabf 2024-11-17T21:25:30,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/85d8a1ff24364e349e69888eca1e4387 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/85d8a1ff24364e349e69888eca1e4387 2024-11-17T21:25:30,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/ebd124ff6fec4060890a81c2baac7d72 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/ebd124ff6fec4060890a81c2baac7d72 2024-11-17T21:25:30,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/9a9d9763d7324207ae25d14b566c838a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/9a9d9763d7324207ae25d14b566c838a 2024-11-17T21:25:30,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/425e319ff944472cbce2af9dee3df1c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/425e319ff944472cbce2af9dee3df1c8 2024-11-17T21:25:30,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8566340f7900401e9038a52f67cb2761 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8566340f7900401e9038a52f67cb2761 2024-11-17T21:25:30,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/2a1bb660584f446e81fae2c5cf0c5126 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/2a1bb660584f446e81fae2c5cf0c5126 2024-11-17T21:25:30,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/106abcb1128b41f18a79c35e34803eaf to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/106abcb1128b41f18a79c35e34803eaf 2024-11-17T21:25:30,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8e2583c13267475daf5f9c38d5b13c79 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8e2583c13267475daf5f9c38d5b13c79 2024-11-17T21:25:30,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/30156ee08fd5424ca7ee51a5954d9e06 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/30156ee08fd5424ca7ee51a5954d9e06 2024-11-17T21:25:30,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/78b0c61a48404f5b9f2d3437390befb5 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/78b0c61a48404f5b9f2d3437390befb5 2024-11-17T21:25:30,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/f17bc6b113f845ddba68a0d0e8665fdc to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/f17bc6b113f845ddba68a0d0e8665fdc 2024-11-17T21:25:30,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/7d7980c5bd3d49a0b064ca1f5a4cb5b7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/7d7980c5bd3d49a0b064ca1f5a4cb5b7 2024-11-17T21:25:30,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/f6106569f5b4451bace82aa26ae02c79 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/f6106569f5b4451bace82aa26ae02c79 2024-11-17T21:25:30,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/54cb5dd6fda34ad780b7226d67ea49bd to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/54cb5dd6fda34ad780b7226d67ea49bd 2024-11-17T21:25:30,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8a6480dc3024479db1287ca273f015bb to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/8a6480dc3024479db1287ca273f015bb 2024-11-17T21:25:30,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/725a6c9c525a4546a5e2d64d80b40610 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/725a6c9c525a4546a5e2d64d80b40610 2024-11-17T21:25:30,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/b43c1405945a47b88639b16f9138d286 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/b43c1405945a47b88639b16f9138d286 2024-11-17T21:25:30,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/12377c6730d644698cb64aa1930bcd13 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/12377c6730d644698cb64aa1930bcd13 2024-11-17T21:25:30,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/5ae8cb7093494298920f25d0292d6cfb to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/5ae8cb7093494298920f25d0292d6cfb 2024-11-17T21:25:30,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/779fae0977944b058f9b8d95023d1a6a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/779fae0977944b058f9b8d95023d1a6a 2024-11-17T21:25:30,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/9440288688494312b2e51757f366bbb3 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/9440288688494312b2e51757f366bbb3 2024-11-17T21:25:30,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/46b4a616bea942e192ea587bbcdc7779, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/413abd33719540829c22cd3c09873faf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/809fcb2b54c84a4d92a6657da549831c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e69148efd2c743249f7923407aba3f25, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/8e0e33ac0ddb4553b6da997ae28869ef, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/bb5db17cbea948a1971c3e9652378927, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7bceb408522c41e0af49f97bcc429e12, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/ffbd350f0f7e49f98e050b72217f7f69, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/d75a438522a14e22a2701c4b581a1027, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/49b9d0a3867143ceaa82de4b73f3b329, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/2b75881f41d647308204c0c3ea28c4be, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e94fe94c4eab42529ec50ff994d1264b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/6e1ae008d6914dfdb5e3bcbf786848aa, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/9572bb03e0344ef28affbbfcc504fa45, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/27a6de79387d4dd1b33328fd2ccd6a09, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e0e0df7f59ba451382c48ed8e00bb6cb, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/c639997e3fb643dfbd988cbf190cafe6, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7779821f961d45a187aae388cb0b622c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7792742aaf614ca7ab37950d5561bb1d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e2653700303c4b22883965f6f683fd2e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/c9b8408c4bca4565b9b81aed9333522e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/6731cad9e0d444d7a8606860b11afeca, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/1fa6a4d199454bb1ad8a69d4cc795e16] to archive 2024-11-17T21:25:30,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:25:30,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/46b4a616bea942e192ea587bbcdc7779 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/46b4a616bea942e192ea587bbcdc7779 2024-11-17T21:25:30,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/413abd33719540829c22cd3c09873faf to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/413abd33719540829c22cd3c09873faf 2024-11-17T21:25:30,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/809fcb2b54c84a4d92a6657da549831c to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/809fcb2b54c84a4d92a6657da549831c 2024-11-17T21:25:30,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e69148efd2c743249f7923407aba3f25 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e69148efd2c743249f7923407aba3f25 2024-11-17T21:25:30,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/8e0e33ac0ddb4553b6da997ae28869ef to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/8e0e33ac0ddb4553b6da997ae28869ef 2024-11-17T21:25:30,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/bb5db17cbea948a1971c3e9652378927 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/bb5db17cbea948a1971c3e9652378927 2024-11-17T21:25:30,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7bceb408522c41e0af49f97bcc429e12 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7bceb408522c41e0af49f97bcc429e12 2024-11-17T21:25:30,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/ffbd350f0f7e49f98e050b72217f7f69 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/ffbd350f0f7e49f98e050b72217f7f69 2024-11-17T21:25:30,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/d75a438522a14e22a2701c4b581a1027 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/d75a438522a14e22a2701c4b581a1027 2024-11-17T21:25:30,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/49b9d0a3867143ceaa82de4b73f3b329 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/49b9d0a3867143ceaa82de4b73f3b329 2024-11-17T21:25:30,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/2b75881f41d647308204c0c3ea28c4be to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/2b75881f41d647308204c0c3ea28c4be 2024-11-17T21:25:30,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e94fe94c4eab42529ec50ff994d1264b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e94fe94c4eab42529ec50ff994d1264b 2024-11-17T21:25:30,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/6e1ae008d6914dfdb5e3bcbf786848aa to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/6e1ae008d6914dfdb5e3bcbf786848aa 2024-11-17T21:25:30,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/9572bb03e0344ef28affbbfcc504fa45 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/9572bb03e0344ef28affbbfcc504fa45 2024-11-17T21:25:30,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/27a6de79387d4dd1b33328fd2ccd6a09 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/27a6de79387d4dd1b33328fd2ccd6a09 2024-11-17T21:25:30,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e0e0df7f59ba451382c48ed8e00bb6cb to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e0e0df7f59ba451382c48ed8e00bb6cb 2024-11-17T21:25:30,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/c639997e3fb643dfbd988cbf190cafe6 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/c639997e3fb643dfbd988cbf190cafe6 2024-11-17T21:25:30,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7779821f961d45a187aae388cb0b622c to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7779821f961d45a187aae388cb0b622c 2024-11-17T21:25:30,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7792742aaf614ca7ab37950d5561bb1d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/7792742aaf614ca7ab37950d5561bb1d 2024-11-17T21:25:30,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e2653700303c4b22883965f6f683fd2e to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/e2653700303c4b22883965f6f683fd2e 2024-11-17T21:25:30,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/c9b8408c4bca4565b9b81aed9333522e to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/c9b8408c4bca4565b9b81aed9333522e 2024-11-17T21:25:30,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/6731cad9e0d444d7a8606860b11afeca to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/6731cad9e0d444d7a8606860b11afeca 2024-11-17T21:25:30,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/1fa6a4d199454bb1ad8a69d4cc795e16 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/1fa6a4d199454bb1ad8a69d4cc795e16 2024-11-17T21:25:30,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/4904d35a5938494b8957e6d59313b1c8, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/88955ed64cfa4abe8dd1181a66d72be3, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/7f441302a8eb4edc9a1594c88c3f4990, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/5b3200fa26ff40ed875084026b82e386, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/b445dc700b844eaab91f7559a66ebfa9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/3713dbc10f874079a0beda02fa62960c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/504166cda3ab4106bde8ce646679b909, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/6e27325accc749d4aad9d6f129cabd3e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/8c98eaa8acd74714a6b5ae54ce5c6ef7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/f68672f3800b400f84edecb1672ec952, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/a855d573906b47ca91c3e0c159f5263d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/9caefd24d06840c1ae63825738f45bbe, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/68843b5cc5b645c7809322fb3b3ad315, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/76a564915b0b4a1e91dc3063a5b71965, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/7b0122af59724b32a5f983b1b2030b44, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/3bb6d4150e0744f28a3336b7a73cc066, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/f4f1b898bafe425f95d3ea233022dee7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/7fa62f223c1f477bb3508c12b71e1ef3, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/1cc1cfd7c6f949e1a5a29119ef650626, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/039bcaf9f52d475ab8378b99df18ba32, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/738b7d0f89f54409b92cc68cd45f403d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/776ea5e6f737473aab965206222c7d82, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/aaf557f433cc4464894dd3222b18fee1] to archive 2024-11-17T21:25:30,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:25:30,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/4904d35a5938494b8957e6d59313b1c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/4904d35a5938494b8957e6d59313b1c8 2024-11-17T21:25:30,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/88955ed64cfa4abe8dd1181a66d72be3 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/88955ed64cfa4abe8dd1181a66d72be3 2024-11-17T21:25:30,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/7f441302a8eb4edc9a1594c88c3f4990 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/7f441302a8eb4edc9a1594c88c3f4990 2024-11-17T21:25:30,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/5b3200fa26ff40ed875084026b82e386 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/5b3200fa26ff40ed875084026b82e386 2024-11-17T21:25:30,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/b445dc700b844eaab91f7559a66ebfa9 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/b445dc700b844eaab91f7559a66ebfa9 2024-11-17T21:25:30,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/3713dbc10f874079a0beda02fa62960c to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/3713dbc10f874079a0beda02fa62960c 2024-11-17T21:25:30,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/504166cda3ab4106bde8ce646679b909 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/504166cda3ab4106bde8ce646679b909 2024-11-17T21:25:30,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/6e27325accc749d4aad9d6f129cabd3e to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/6e27325accc749d4aad9d6f129cabd3e 2024-11-17T21:25:30,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/8c98eaa8acd74714a6b5ae54ce5c6ef7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/8c98eaa8acd74714a6b5ae54ce5c6ef7 2024-11-17T21:25:30,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/f68672f3800b400f84edecb1672ec952 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/f68672f3800b400f84edecb1672ec952 2024-11-17T21:25:30,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/a855d573906b47ca91c3e0c159f5263d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/a855d573906b47ca91c3e0c159f5263d 2024-11-17T21:25:30,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/9caefd24d06840c1ae63825738f45bbe to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/9caefd24d06840c1ae63825738f45bbe 2024-11-17T21:25:30,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/68843b5cc5b645c7809322fb3b3ad315 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/68843b5cc5b645c7809322fb3b3ad315 2024-11-17T21:25:30,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/76a564915b0b4a1e91dc3063a5b71965 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/76a564915b0b4a1e91dc3063a5b71965 2024-11-17T21:25:30,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/7b0122af59724b32a5f983b1b2030b44 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/7b0122af59724b32a5f983b1b2030b44 2024-11-17T21:25:30,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/3bb6d4150e0744f28a3336b7a73cc066 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/3bb6d4150e0744f28a3336b7a73cc066 2024-11-17T21:25:30,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/f4f1b898bafe425f95d3ea233022dee7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/f4f1b898bafe425f95d3ea233022dee7 2024-11-17T21:25:30,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/7fa62f223c1f477bb3508c12b71e1ef3 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/7fa62f223c1f477bb3508c12b71e1ef3 2024-11-17T21:25:30,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/1cc1cfd7c6f949e1a5a29119ef650626 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/1cc1cfd7c6f949e1a5a29119ef650626 2024-11-17T21:25:30,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/039bcaf9f52d475ab8378b99df18ba32 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/039bcaf9f52d475ab8378b99df18ba32 2024-11-17T21:25:30,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/738b7d0f89f54409b92cc68cd45f403d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/738b7d0f89f54409b92cc68cd45f403d 2024-11-17T21:25:30,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/776ea5e6f737473aab965206222c7d82 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/776ea5e6f737473aab965206222c7d82 2024-11-17T21:25:30,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/aaf557f433cc4464894dd3222b18fee1 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/aaf557f433cc4464894dd3222b18fee1 2024-11-17T21:25:30,595 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/recovered.edits/376.seqid, newMaxSeqId=376, maxSeqId=1 2024-11-17T21:25:30,596 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. 2024-11-17T21:25:30,596 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 224e951b47a00740dfea2342be622072: Waiting for close lock at 1731878728652Running coprocessor pre-close hooks at 1731878728652Disabling compacts and flushes for region at 1731878728652Disabling writes for close at 1731878729257 (+605 ms)Obtaining lock to block concurrent updates at 1731878729257Preparing flush snapshotting stores in 224e951b47a00740dfea2342be622072 at 1731878729257Finished memstore snapshotting TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731878729257Flushing stores of TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072. at 1731878729258 (+1 ms)Flushing 224e951b47a00740dfea2342be622072/A: creating writer at 1731878729258Flushing 224e951b47a00740dfea2342be622072/A: appending metadata at 1731878729259 (+1 ms)Flushing 224e951b47a00740dfea2342be622072/A: closing flushed file at 1731878729259Flushing 224e951b47a00740dfea2342be622072/B: creating writer at 1731878729674 (+415 ms)Flushing 224e951b47a00740dfea2342be622072/B: appending metadata at 1731878729678 (+4 ms)Flushing 224e951b47a00740dfea2342be622072/B: closing flushed file at 1731878729678Flushing 224e951b47a00740dfea2342be622072/C: creating writer at 1731878730097 (+419 ms)Flushing 224e951b47a00740dfea2342be622072/C: appending metadata at 1731878730099 (+2 ms)Flushing 224e951b47a00740dfea2342be622072/C: closing flushed file at 1731878730099Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43b84d9f: reopening flushed file at 1731878730514 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@765d9459: reopening flushed file at 1731878730523 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57dc296a: reopening flushed file at 1731878730528 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 224e951b47a00740dfea2342be622072 in 1276ms, sequenceid=373, compaction requested=true at 1731878730533 (+5 ms)Writing region close event to WAL at 1731878730592 (+59 ms)Running coprocessor post-close hooks at 1731878730596 (+4 ms)Closed at 1731878730596 2024-11-17T21:25:30,598 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 224e951b47a00740dfea2342be622072 2024-11-17T21:25:30,598 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=224e951b47a00740dfea2342be622072, regionState=CLOSED 2024-11-17T21:25:30,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:25:30,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-17T21:25:30,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 224e951b47a00740dfea2342be622072, server=a313eea8709e,46445,1731878642828 in 2.1010 sec 2024-11-17T21:25:30,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-17T21:25:30,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=224e951b47a00740dfea2342be622072, UNASSIGN in 2.1050 sec 2024-11-17T21:25:30,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-17T21:25:30,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1090 sec 2024-11-17T21:25:30,606 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878730605"}]},"ts":"1731878730605"} 2024-11-17T21:25:30,607 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T21:25:30,607 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T21:25:30,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1230 sec 2024-11-17T21:25:30,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T21:25:30,641 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:30,642 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T21:25:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:30,643 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T21:25:30,644 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:30,646 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072 2024-11-17T21:25:30,648 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/recovered.edits] 2024-11-17T21:25:30,651 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/7b4c3720b68d4409adf2674fc51fd381 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/7b4c3720b68d4409adf2674fc51fd381 2024-11-17T21:25:30,652 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/d529d618b7e34a1fbc5ec33ca8aae189 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/d529d618b7e34a1fbc5ec33ca8aae189 2024-11-17T21:25:30,653 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/de110c4bde5b46469e5ca5867c727168 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/de110c4bde5b46469e5ca5867c727168 2024-11-17T21:25:30,654 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/fb4d76c6441142d2a60f34ae6ae486a4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/A/fb4d76c6441142d2a60f34ae6ae486a4 2024-11-17T21:25:30,656 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/18eadad8918b4cd8b133816465458d4d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/18eadad8918b4cd8b133816465458d4d 2024-11-17T21:25:30,657 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/38cf412b24404cba9db56b50984a5373 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/38cf412b24404cba9db56b50984a5373 2024-11-17T21:25:30,658 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/bb35d7afb6034bf5b8c68fb977dffc13 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/bb35d7afb6034bf5b8c68fb977dffc13 2024-11-17T21:25:30,659 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/dc3f2fe0de014f139a316cf29fd0c3ab to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/B/dc3f2fe0de014f139a316cf29fd0c3ab 2024-11-17T21:25:30,661 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/228350aa200b468ab5d70d0ac708a740 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/228350aa200b468ab5d70d0ac708a740 2024-11-17T21:25:30,662 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/4a48bfec3cf14c77be12da1d141e5e37 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/4a48bfec3cf14c77be12da1d141e5e37 2024-11-17T21:25:30,663 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/84b6be5b2fa14199b2064703e675aad7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/84b6be5b2fa14199b2064703e675aad7 2024-11-17T21:25:30,664 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/8e2f05e05289410591b25881d0af2086 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/C/8e2f05e05289410591b25881d0af2086 2024-11-17T21:25:30,666 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/recovered.edits/376.seqid to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072/recovered.edits/376.seqid 2024-11-17T21:25:30,667 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/224e951b47a00740dfea2342be622072 2024-11-17T21:25:30,667 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T21:25:30,669 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:30,672 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T21:25:30,674 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T21:25:30,675 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:30,675 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T21:25:30,676 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731878730675"}]},"ts":"9223372036854775807"} 2024-11-17T21:25:30,677 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T21:25:30,677 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 224e951b47a00740dfea2342be622072, NAME => 'TestAcidGuarantees,,1731878705286.224e951b47a00740dfea2342be622072.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T21:25:30,677 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T21:25:30,678 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731878730677"}]},"ts":"9223372036854775807"} 2024-11-17T21:25:30,679 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T21:25:30,680 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:30,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-17T21:25:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T21:25:30,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T21:25:30,751 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:30,760 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 241), OpenFileDescriptor=451 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=547 (was 451) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8323 (was 8160) - AvailableMemoryMB LEAK? - 2024-11-17T21:25:30,770 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=547, ProcessCount=11, AvailableMemoryMB=8322 2024-11-17T21:25:30,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T21:25:30,771 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T21:25:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:30,773 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T21:25:30,773 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:30,774 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-17T21:25:30,774 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T21:25:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T21:25:30,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742156_1332 (size=959) 2024-11-17T21:25:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T21:25:31,090 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T21:25:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T21:25:31,183 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249 2024-11-17T21:25:31,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742157_1333 (size=53) 2024-11-17T21:25:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T21:25:31,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:25:31,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c3ee12f1af122429c9b29c87eab606c8, disabling compactions & flushes 2024-11-17T21:25:31,593 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:31,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:31,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. after waiting 0 ms 2024-11-17T21:25:31,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:31,593 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:31,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c3ee12f1af122429c9b29c87eab606c8: Waiting for close lock at 1731878731593Disabling compacts and flushes for region at 1731878731593Disabling writes for close at 1731878731593Writing region close event to WAL at 1731878731593Closed at 1731878731593 2024-11-17T21:25:31,595 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T21:25:31,596 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731878731595"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731878731595"}]},"ts":"1731878731595"} 2024-11-17T21:25:31,598 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T21:25:31,600 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T21:25:31,600 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878731600"}]},"ts":"1731878731600"} 2024-11-17T21:25:31,602 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T21:25:31,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c3ee12f1af122429c9b29c87eab606c8, ASSIGN}] 2024-11-17T21:25:31,603 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c3ee12f1af122429c9b29c87eab606c8, ASSIGN 2024-11-17T21:25:31,604 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c3ee12f1af122429c9b29c87eab606c8, ASSIGN; state=OFFLINE, location=a313eea8709e,46445,1731878642828; forceNewPlan=false, retain=false 2024-11-17T21:25:31,756 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=c3ee12f1af122429c9b29c87eab606c8, regionState=OPENING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:25:31,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c3ee12f1af122429c9b29c87eab606c8, ASSIGN because future has completed 2024-11-17T21:25:31,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:25:31,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T21:25:31,921 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:31,922 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} 2024-11-17T21:25:31,922 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:31,922 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:25:31,923 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:31,923 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:31,925 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:31,928 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3ee12f1af122429c9b29c87eab606c8 columnFamilyName A 2024-11-17T21:25:31,928 DEBUG [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:31,929 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] regionserver.HStore(327): Store=c3ee12f1af122429c9b29c87eab606c8/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:25:31,930 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:31,932 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3ee12f1af122429c9b29c87eab606c8 columnFamilyName B 2024-11-17T21:25:31,932 DEBUG [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:31,933 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] regionserver.HStore(327): Store=c3ee12f1af122429c9b29c87eab606c8/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:25:31,933 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:31,935 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3ee12f1af122429c9b29c87eab606c8 columnFamilyName C 2024-11-17T21:25:31,935 DEBUG [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:31,936 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] regionserver.HStore(327): Store=c3ee12f1af122429c9b29c87eab606c8/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:25:31,936 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:31,937 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:31,937 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:31,938 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:31,938 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:31,938 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T21:25:31,940 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:31,941 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T21:25:31,942 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened c3ee12f1af122429c9b29c87eab606c8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72183095, jitterRate=0.07561193406581879}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T21:25:31,942 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:31,943 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for c3ee12f1af122429c9b29c87eab606c8: Running coprocessor pre-open hook at 1731878731923Writing region info on filesystem at 1731878731923Initializing all the Stores at 1731878731925 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878731925Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878731925Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878731925Cleaning up temporary data from old regions at 1731878731938 (+13 ms)Running coprocessor post-open hooks at 1731878731942 (+4 ms)Region opened successfully at 1731878731943 (+1 ms) 2024-11-17T21:25:31,944 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., pid=87, masterSystemTime=1731878731915 2024-11-17T21:25:31,946 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:31,946 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:31,946 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=c3ee12f1af122429c9b29c87eab606c8, regionState=OPEN, openSeqNum=2, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:25:31,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:25:31,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-17T21:25:31,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 in 188 msec 2024-11-17T21:25:31,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-17T21:25:31,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c3ee12f1af122429c9b29c87eab606c8, ASSIGN in 348 msec 2024-11-17T21:25:31,953 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T21:25:31,953 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878731953"}]},"ts":"1731878731953"} 2024-11-17T21:25:31,954 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T21:25:31,955 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T21:25:31,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-11-17T21:25:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T21:25:32,931 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:32,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1150892c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:32,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:32,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:32,933 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:32,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:32,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:32,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@444f91a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:32,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:32,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:32,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:32,935 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c7117e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:32,936 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:32,938 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:32,939 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T21:25:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is a313eea8709e,43623,1731878641945 2024-11-17T21:25:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@30424fe8 2024-11-17T21:25:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T21:25:32,941 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T21:25:32,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T21:25:32,942 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T21:25:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T21:25:32,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742158_1334 (size=995) 2024-11-17T21:25:33,357 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-17T21:25:33,357 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-17T21:25:33,361 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T21:25:33,363 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c3ee12f1af122429c9b29c87eab606c8, REOPEN/MOVE}] 2024-11-17T21:25:33,364 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c3ee12f1af122429c9b29c87eab606c8, REOPEN/MOVE 2024-11-17T21:25:33,365 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=c3ee12f1af122429c9b29c87eab606c8, regionState=CLOSING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:25:33,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c3ee12f1af122429c9b29c87eab606c8, REOPEN/MOVE because future has completed 2024-11-17T21:25:33,366 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T21:25:33,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:25:33,519 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:33,519 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T21:25:33,519 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing c3ee12f1af122429c9b29c87eab606c8, disabling compactions & flushes 2024-11-17T21:25:33,519 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:33,519 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:33,519 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. after waiting 0 ms 2024-11-17T21:25:33,519 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:33,524 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T21:25:33,525 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:33,525 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for c3ee12f1af122429c9b29c87eab606c8: Waiting for close lock at 1731878733519Running coprocessor pre-close hooks at 1731878733519Disabling compacts and flushes for region at 1731878733519Disabling writes for close at 1731878733519Writing region close event to WAL at 1731878733520 (+1 ms)Running coprocessor post-close hooks at 1731878733525 (+5 ms)Closed at 1731878733525 2024-11-17T21:25:33,525 WARN [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: c3ee12f1af122429c9b29c87eab606c8 to self. 2024-11-17T21:25:33,527 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:33,527 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=c3ee12f1af122429c9b29c87eab606c8, regionState=CLOSED 2024-11-17T21:25:33,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:25:33,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-17T21:25:33,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 in 163 msec 2024-11-17T21:25:33,531 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c3ee12f1af122429c9b29c87eab606c8, REOPEN/MOVE; state=CLOSED, location=a313eea8709e,46445,1731878642828; forceNewPlan=false, retain=true 2024-11-17T21:25:33,682 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=c3ee12f1af122429c9b29c87eab606c8, regionState=OPENING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:25:33,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c3ee12f1af122429c9b29c87eab606c8, REOPEN/MOVE because future has completed 2024-11-17T21:25:33,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:25:33,838 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:33,838 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} 2024-11-17T21:25:33,838 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:33,838 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:25:33,838 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:33,838 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:33,840 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:33,841 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3ee12f1af122429c9b29c87eab606c8 columnFamilyName A 2024-11-17T21:25:33,841 DEBUG [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:33,842 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] regionserver.HStore(327): Store=c3ee12f1af122429c9b29c87eab606c8/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:25:33,842 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:33,843 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3ee12f1af122429c9b29c87eab606c8 columnFamilyName B 2024-11-17T21:25:33,843 DEBUG [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:33,843 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] regionserver.HStore(327): Store=c3ee12f1af122429c9b29c87eab606c8/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:25:33,843 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:33,844 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3ee12f1af122429c9b29c87eab606c8 columnFamilyName C 2024-11-17T21:25:33,844 DEBUG [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:33,844 INFO [StoreOpener-c3ee12f1af122429c9b29c87eab606c8-1 {}] regionserver.HStore(327): Store=c3ee12f1af122429c9b29c87eab606c8/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:25:33,844 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:33,845 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:33,845 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:33,846 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:33,846 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:33,847 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T21:25:33,848 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:33,848 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened c3ee12f1af122429c9b29c87eab606c8; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75495544, jitterRate=0.12497127056121826}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T21:25:33,848 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:33,849 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for c3ee12f1af122429c9b29c87eab606c8: Running coprocessor pre-open hook at 1731878733839Writing region info on filesystem at 1731878733839Initializing all the Stores at 1731878733839Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878733839Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878733840 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878733840Cleaning up temporary data from old regions at 1731878733846 (+6 ms)Running coprocessor post-open hooks at 1731878733848 (+2 ms)Region opened successfully at 1731878733849 (+1 ms) 2024-11-17T21:25:33,850 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., pid=92, masterSystemTime=1731878733835 2024-11-17T21:25:33,851 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:33,851 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:33,852 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=c3ee12f1af122429c9b29c87eab606c8, regionState=OPEN, openSeqNum=5, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:25:33,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:25:33,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-17T21:25:33,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 in 170 msec 2024-11-17T21:25:33,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-17T21:25:33,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c3ee12f1af122429c9b29c87eab606c8, REOPEN/MOVE in 492 msec 2024-11-17T21:25:33,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-17T21:25:33,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 497 msec 2024-11-17T21:25:33,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 917 msec 2024-11-17T21:25:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T21:25:33,861 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:33,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f86ed2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,861 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:33,862 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:33,862 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:33,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:33,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:33,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b20a8b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:33,863 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d8cfd94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59f0df0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,864 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:33,864 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:33,865 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e07fa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:33,866 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:33,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@743d11ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@927442d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,867 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:33,867 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:33,868 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e4a7328, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:33,869 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e5d808, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@725928f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:33,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:33,870 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5859d203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:33,871 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@216e1c49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eab4bc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:33,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:33,873 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a939a6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:33,874 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:33,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1688406d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@580773ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:33,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:33,876 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:33,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:33,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:33,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1751152, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:33,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:33,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:33,877 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:33,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e59164b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34dcef37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:33,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:33,878 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@785bd7a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:33,879 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:33,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19614cbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5713b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:33,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:33,881 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:33,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:33,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0b743, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:33,882 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180d0da7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d2483a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:33,883 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:33,884 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@731a7679, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:33,886 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f95956d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b3d900, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:25:33,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:25:33,887 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:25:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:25:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:25:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1978d5b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:25:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:25:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:33,889 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:25:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4472bacb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:25:33,896 DEBUG [Thread-1528 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:33,896 DEBUG [Thread-1543 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:33,896 DEBUG [Thread-1541 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:33,896 DEBUG [Thread-1532 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:33,897 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:25:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:33,897 DEBUG [Thread-1530 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:33,898 DEBUG [Thread-1537 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:33,898 DEBUG [Thread-1539 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-17T21:25:33,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:33,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:33,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T21:25:33,900 DEBUG [Thread-1526 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:33,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:33,900 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:25:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:33,901 DEBUG [Thread-1526 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:33,901 DEBUG [Thread-1526 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:33,901 DEBUG [Thread-1539 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:33,901 DEBUG [Thread-1539 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:33,902 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:25:33,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:25:33,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:33,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:33,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:33,903 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:33,903 DEBUG [Thread-1524 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:33,903 DEBUG [Thread-1535 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:25:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:25:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:25:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:25:33,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:33,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:25:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:25:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:25:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:25:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:25:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:25:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:25:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:25:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:25:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:25:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:33,918 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:33,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:25:33,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117749688b9346340b7a70700f3b0373c7e_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878733918/Put/seqid=0 2024-11-17T21:25:33,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878793968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:33,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878793970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:33,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:33,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:33,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:33,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878793975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:33,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:33,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:33,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:33,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878793982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:33,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:33,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:33,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:33,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878793983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:33,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:33,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:33,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742159_1335 (size=14594) 2024-11-17T21:25:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T21:25:34,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T21:25:34,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:34,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:34,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:34,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878794091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878794091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878794101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878794101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878794101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T21:25:34,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:34,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:34,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:34,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T21:25:34,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878794301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878794301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878794312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878794312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878794312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T21:25:34,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:34,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:34,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:34,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,407 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:34,414 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117749688b9346340b7a70700f3b0373c7e_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117749688b9346340b7a70700f3b0373c7e_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:34,417 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/c47bb3ebde3245858a06b2d1ae3295ff, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:34,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/c47bb3ebde3245858a06b2d1ae3295ff is 175, key is test_row_0/A:col10/1731878733918/Put/seqid=0 2024-11-17T21:25:34,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742160_1336 (size=39549) 2024-11-17T21:25:34,436 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/c47bb3ebde3245858a06b2d1ae3295ff 2024-11-17T21:25:34,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/8c9a24af8a3f4a1586c6b6320236bb15 is 50, key is test_row_0/B:col10/1731878733918/Put/seqid=0 2024-11-17T21:25:34,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742161_1337 (size=12001) 2024-11-17T21:25:34,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/8c9a24af8a3f4a1586c6b6320236bb15 2024-11-17T21:25:34,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T21:25:34,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:34,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:34,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:34,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/c15be5fb8657466abcd37ee0701964a7 is 50, key is test_row_0/C:col10/1731878733918/Put/seqid=0 2024-11-17T21:25:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T21:25:34,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742162_1338 (size=12001) 2024-11-17T21:25:34,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/c15be5fb8657466abcd37ee0701964a7 2024-11-17T21:25:34,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/c47bb3ebde3245858a06b2d1ae3295ff as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c47bb3ebde3245858a06b2d1ae3295ff 2024-11-17T21:25:34,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c47bb3ebde3245858a06b2d1ae3295ff, entries=200, sequenceid=17, filesize=38.6 K 2024-11-17T21:25:34,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/8c9a24af8a3f4a1586c6b6320236bb15 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/8c9a24af8a3f4a1586c6b6320236bb15 2024-11-17T21:25:34,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/8c9a24af8a3f4a1586c6b6320236bb15, entries=150, sequenceid=17, filesize=11.7 K 2024-11-17T21:25:34,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/c15be5fb8657466abcd37ee0701964a7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/c15be5fb8657466abcd37ee0701964a7 2024-11-17T21:25:34,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/c15be5fb8657466abcd37ee0701964a7, entries=150, sequenceid=17, filesize=11.7 K 2024-11-17T21:25:34,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c3ee12f1af122429c9b29c87eab606c8 in 665ms, sequenceid=17, compaction requested=false 2024-11-17T21:25:34,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:34,628 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:34,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T21:25:34,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172c16ce30bcad427db013577b245e3cd5_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878733979/Put/seqid=0 2024-11-17T21:25:34,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878794646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878794647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878794650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878794651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878794655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T21:25:34,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:34,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:34,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:34,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742163_1339 (size=17034) 2024-11-17T21:25:34,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878794771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878794771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878794771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878794771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878794782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:34,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T21:25:34,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:34,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:34,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:34,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T21:25:34,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:34,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:34,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:34,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:35,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878794991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878794991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:35,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878794991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:35,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878794992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:35,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878795001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:35,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T21:25:35,104 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:35,107 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172c16ce30bcad427db013577b245e3cd5_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172c16ce30bcad427db013577b245e3cd5_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:35,108 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/5960681efc3d4beebb062d81b2b0fa4c, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:35,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/5960681efc3d4beebb062d81b2b0fa4c is 175, key is test_row_0/A:col10/1731878733979/Put/seqid=0 2024-11-17T21:25:35,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T21:25:35,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:35,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:35,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:35,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:35,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:35,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742164_1340 (size=48139) 2024-11-17T21:25:35,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T21:25:35,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:35,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:35,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:35,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:35,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:35,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878795311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878795311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:35,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878795311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:35,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878795312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:35,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:35,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878795321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:35,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T21:25:35,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:35,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:35,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:35,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:35,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:35,542 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=43, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/5960681efc3d4beebb062d81b2b0fa4c 2024-11-17T21:25:35,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/a51894c33d574a1c8a055ff434e0431c is 50, key is test_row_0/B:col10/1731878733979/Put/seqid=0 2024-11-17T21:25:35,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742165_1341 (size=12001) 2024-11-17T21:25:35,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/a51894c33d574a1c8a055ff434e0431c 2024-11-17T21:25:35,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/4e07432d038d43b6a59d5f912eea0fcd is 50, key is test_row_0/C:col10/1731878733979/Put/seqid=0 2024-11-17T21:25:35,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T21:25:35,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:35,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:35,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:35,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:35,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:35,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742166_1342 (size=12001) 2024-11-17T21:25:35,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/4e07432d038d43b6a59d5f912eea0fcd 2024-11-17T21:25:35,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/5960681efc3d4beebb062d81b2b0fa4c as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/5960681efc3d4beebb062d81b2b0fa4c 2024-11-17T21:25:35,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/5960681efc3d4beebb062d81b2b0fa4c, entries=250, sequenceid=43, filesize=47.0 K 2024-11-17T21:25:35,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/a51894c33d574a1c8a055ff434e0431c as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/a51894c33d574a1c8a055ff434e0431c 2024-11-17T21:25:35,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/a51894c33d574a1c8a055ff434e0431c, entries=150, sequenceid=43, filesize=11.7 K 2024-11-17T21:25:35,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/4e07432d038d43b6a59d5f912eea0fcd as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/4e07432d038d43b6a59d5f912eea0fcd 2024-11-17T21:25:35,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/4e07432d038d43b6a59d5f912eea0fcd, entries=150, sequenceid=43, filesize=11.7 K 2024-11-17T21:25:35,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for c3ee12f1af122429c9b29c87eab606c8 in 1013ms, sequenceid=43, compaction requested=false 2024-11-17T21:25:35,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:35,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T21:25:35,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:35,762 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-17T21:25:35,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111741aa6d4b284a411b87cf8d14f8652f87_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878734652/Put/seqid=0 2024-11-17T21:25:35,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742167_1343 (size=12154) 2024-11-17T21:25:35,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:35,788 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111741aa6d4b284a411b87cf8d14f8652f87_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111741aa6d4b284a411b87cf8d14f8652f87_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:35,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/ecf9ea3a103f427fa145753664c0c593, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:35,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/ecf9ea3a103f427fa145753664c0c593 is 175, key is test_row_0/A:col10/1731878734652/Put/seqid=0 2024-11-17T21:25:35,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742168_1344 (size=30955) 2024-11-17T21:25:35,801 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/ecf9ea3a103f427fa145753664c0c593 2024-11-17T21:25:35,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/ebe6e4b9fee945e888e7b832c74cefc1 is 50, key is test_row_0/B:col10/1731878734652/Put/seqid=0 2024-11-17T21:25:35,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742169_1345 (size=12001) 2024-11-17T21:25:35,816 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/ebe6e4b9fee945e888e7b832c74cefc1 2024-11-17T21:25:35,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/14f7bf4893e24f2490c0eed9dc193bd3 is 50, key is test_row_0/C:col10/1731878734652/Put/seqid=0 2024-11-17T21:25:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:35,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742170_1346 (size=12001) 2024-11-17T21:25:35,835 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/14f7bf4893e24f2490c0eed9dc193bd3 2024-11-17T21:25:35,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/ecf9ea3a103f427fa145753664c0c593 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/ecf9ea3a103f427fa145753664c0c593 2024-11-17T21:25:35,847 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/ecf9ea3a103f427fa145753664c0c593, entries=150, sequenceid=53, filesize=30.2 K 2024-11-17T21:25:35,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/ebe6e4b9fee945e888e7b832c74cefc1 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ebe6e4b9fee945e888e7b832c74cefc1 2024-11-17T21:25:35,853 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ebe6e4b9fee945e888e7b832c74cefc1, entries=150, sequenceid=53, filesize=11.7 K 2024-11-17T21:25:35,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/14f7bf4893e24f2490c0eed9dc193bd3 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/14f7bf4893e24f2490c0eed9dc193bd3 2024-11-17T21:25:35,872 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/14f7bf4893e24f2490c0eed9dc193bd3, entries=150, sequenceid=53, filesize=11.7 K 2024-11-17T21:25:35,874 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=127.47 KB/130530 for c3ee12f1af122429c9b29c87eab606c8 in 111ms, sequenceid=53, compaction requested=true 2024-11-17T21:25:35,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:35,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:35,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-17T21:25:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-17T21:25:35,877 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-17T21:25:35,877 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9730 sec 2024-11-17T21:25:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:35,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.9810 sec 2024-11-17T21:25:35,880 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:35,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T21:25:35,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d45ce956296a4150b653507fa521b12b_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878735878/Put/seqid=0 2024-11-17T21:25:35,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878795897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:35,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878795902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:35,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:35,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878795902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:35,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:35,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878795903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878795912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:35,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742171_1347 (size=14594) 2024-11-17T21:25:36,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878796021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878796021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878796022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878796022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878796022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T21:25:36,051 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:36,051 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:25:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-17T21:25:36,054 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:25:36,056 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:25:36,056 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:25:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T21:25:36,128 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T21:25:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T21:25:36,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-17T21:25:36,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:36,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:36,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:36,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:36,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:36,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:36,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878796241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878796241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878796242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878796242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878796242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,322 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:36,325 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d45ce956296a4150b653507fa521b12b_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d45ce956296a4150b653507fa521b12b_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:36,326 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/a59390235d634245b2e5dba4abedd33d, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:36,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/a59390235d634245b2e5dba4abedd33d is 175, key is test_row_0/A:col10/1731878735878/Put/seqid=0 2024-11-17T21:25:36,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742172_1348 (size=39549) 2024-11-17T21:25:36,331 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/a59390235d634245b2e5dba4abedd33d 2024-11-17T21:25:36,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/4ffc3dfe4b3c49b7a5676b13f13c588b is 50, key is test_row_0/B:col10/1731878735878/Put/seqid=0 2024-11-17T21:25:36,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742173_1349 (size=12001) 2024-11-17T21:25:36,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/4ffc3dfe4b3c49b7a5676b13f13c588b 2024-11-17T21:25:36,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-17T21:25:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:36,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T21:25:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:36,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/da6564d5050746f0be4a0e346a8aac5b is 50, key is test_row_0/C:col10/1731878735878/Put/seqid=0 2024-11-17T21:25:36,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742174_1350 (size=12001) 2024-11-17T21:25:36,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/da6564d5050746f0be4a0e346a8aac5b 2024-11-17T21:25:36,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/a59390235d634245b2e5dba4abedd33d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/a59390235d634245b2e5dba4abedd33d 2024-11-17T21:25:36,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/a59390235d634245b2e5dba4abedd33d, entries=200, sequenceid=78, filesize=38.6 K 2024-11-17T21:25:36,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/4ffc3dfe4b3c49b7a5676b13f13c588b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/4ffc3dfe4b3c49b7a5676b13f13c588b 2024-11-17T21:25:36,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/4ffc3dfe4b3c49b7a5676b13f13c588b, entries=150, sequenceid=78, filesize=11.7 K 2024-11-17T21:25:36,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/da6564d5050746f0be4a0e346a8aac5b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/da6564d5050746f0be4a0e346a8aac5b 2024-11-17T21:25:36,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/da6564d5050746f0be4a0e346a8aac5b, entries=150, sequenceid=78, filesize=11.7 K 2024-11-17T21:25:36,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c3ee12f1af122429c9b29c87eab606c8 in 556ms, sequenceid=78, compaction requested=true 2024-11-17T21:25:36,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:36,436 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:36,438 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:36,438 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/A is initiating minor compaction (all files) 2024-11-17T21:25:36,438 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/A in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:36,438 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c47bb3ebde3245858a06b2d1ae3295ff, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/5960681efc3d4beebb062d81b2b0fa4c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/ecf9ea3a103f427fa145753664c0c593, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/a59390235d634245b2e5dba4abedd33d] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=154.5 K 2024-11-17T21:25:36,438 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:36,438 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c47bb3ebde3245858a06b2d1ae3295ff, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/5960681efc3d4beebb062d81b2b0fa4c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/ecf9ea3a103f427fa145753664c0c593, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/a59390235d634245b2e5dba4abedd33d] 2024-11-17T21:25:36,439 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting c47bb3ebde3245858a06b2d1ae3295ff, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731878733912 2024-11-17T21:25:36,439 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5960681efc3d4beebb062d81b2b0fa4c, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731878733979 2024-11-17T21:25:36,439 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecf9ea3a103f427fa145753664c0c593, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731878734647 2024-11-17T21:25:36,440 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting a59390235d634245b2e5dba4abedd33d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731878735870 2024-11-17T21:25:36,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:25:36,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:36,444 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:36,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:36,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:36,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:25:36,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:36,451 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:36,451 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/B is initiating minor compaction (all files) 2024-11-17T21:25:36,451 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/B in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:36,451 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/8c9a24af8a3f4a1586c6b6320236bb15, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/a51894c33d574a1c8a055ff434e0431c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ebe6e4b9fee945e888e7b832c74cefc1, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/4ffc3dfe4b3c49b7a5676b13f13c588b] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=46.9 K 2024-11-17T21:25:36,454 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c9a24af8a3f4a1586c6b6320236bb15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731878733916 2024-11-17T21:25:36,454 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting a51894c33d574a1c8a055ff434e0431c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731878733979 2024-11-17T21:25:36,455 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting ebe6e4b9fee945e888e7b832c74cefc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731878734647 2024-11-17T21:25:36,455 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ffc3dfe4b3c49b7a5676b13f13c588b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731878735870 2024-11-17T21:25:36,461 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:36,480 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#B#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:36,481 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/2053d16cfb6349a7bdd6855764da2999 is 50, key is test_row_0/B:col10/1731878735878/Put/seqid=0 2024-11-17T21:25:36,485 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117bd1bcc75a6084562938b8588853d5ed7_c3ee12f1af122429c9b29c87eab606c8 store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:36,490 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117bd1bcc75a6084562938b8588853d5ed7_c3ee12f1af122429c9b29c87eab606c8, store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:36,492 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bd1bcc75a6084562938b8588853d5ed7_c3ee12f1af122429c9b29c87eab606c8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:36,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742175_1351 (size=12139) 2024-11-17T21:25:36,517 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/2053d16cfb6349a7bdd6855764da2999 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/2053d16cfb6349a7bdd6855764da2999 2024-11-17T21:25:36,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-17T21:25:36,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:36,524 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:25:36,526 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/B of c3ee12f1af122429c9b29c87eab606c8 into 2053d16cfb6349a7bdd6855764da2999(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:36,526 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:36,526 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/B, priority=12, startTime=1731878736444; duration=0sec 2024-11-17T21:25:36,526 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:36,526 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:B 2024-11-17T21:25:36,526 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:36,527 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:36,527 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/C is initiating minor compaction (all files) 2024-11-17T21:25:36,528 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/C in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:36,528 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/c15be5fb8657466abcd37ee0701964a7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/4e07432d038d43b6a59d5f912eea0fcd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/14f7bf4893e24f2490c0eed9dc193bd3, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/da6564d5050746f0be4a0e346a8aac5b] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=46.9 K 2024-11-17T21:25:36,528 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting c15be5fb8657466abcd37ee0701964a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731878733916 2024-11-17T21:25:36,528 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e07432d038d43b6a59d5f912eea0fcd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731878733979 2024-11-17T21:25:36,528 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 14f7bf4893e24f2490c0eed9dc193bd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731878734647 2024-11-17T21:25:36,529 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting da6564d5050746f0be4a0e346a8aac5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731878735870 2024-11-17T21:25:36,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742176_1352 (size=4469) 2024-11-17T21:25:36,559 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#C#compaction#301 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:36,559 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/f46fa0e3d8ee469b87208b9aa85bb9af is 50, key is test_row_0/C:col10/1731878735878/Put/seqid=0 2024-11-17T21:25:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:36,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:36,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742177_1353 (size=12139) 2024-11-17T21:25:36,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b7db32072c324ade9b80b7a737678d8a_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878735909/Put/seqid=0 2024-11-17T21:25:36,591 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/f46fa0e3d8ee469b87208b9aa85bb9af as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/f46fa0e3d8ee469b87208b9aa85bb9af 2024-11-17T21:25:36,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742178_1354 (size=12154) 2024-11-17T21:25:36,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:36,601 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/C of c3ee12f1af122429c9b29c87eab606c8 into f46fa0e3d8ee469b87208b9aa85bb9af(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:36,601 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:36,601 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/C, priority=12, startTime=1731878736444; duration=0sec 2024-11-17T21:25:36,602 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:36,602 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:C 2024-11-17T21:25:36,604 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b7db32072c324ade9b80b7a737678d8a_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b7db32072c324ade9b80b7a737678d8a_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:36,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/c2187f376ddd4904ae35a41119d49522, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:36,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/c2187f376ddd4904ae35a41119d49522 is 175, key is test_row_0/A:col10/1731878735909/Put/seqid=0 2024-11-17T21:25:36,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742179_1355 (size=30955) 2024-11-17T21:25:36,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878796626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878796627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878796628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878796629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878796631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T21:25:36,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878796751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878796751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878796751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878796751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878796751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,938 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#A#compaction#299 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:36,939 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/f609e31113424eafaebf8914ffd9d9df is 175, key is test_row_0/A:col10/1731878735878/Put/seqid=0 2024-11-17T21:25:36,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742180_1356 (size=31093) 2024-11-17T21:25:36,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878796961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878796961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878796961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878796971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:36,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878796971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:36,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:36,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:37,032 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/c2187f376ddd4904ae35a41119d49522 2024-11-17T21:25:37,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/8f95d7f5bc4245ebbe02faec13314310 is 50, key is test_row_0/B:col10/1731878735909/Put/seqid=0 2024-11-17T21:25:37,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742181_1357 (size=12001) 2024-11-17T21:25:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T21:25:37,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878797271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:37,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878797271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:37,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878797280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:37,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:37,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878797281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:37,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:37,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878797281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:37,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:37,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:37,349 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/f609e31113424eafaebf8914ffd9d9df as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/f609e31113424eafaebf8914ffd9d9df 2024-11-17T21:25:37,356 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/A of c3ee12f1af122429c9b29c87eab606c8 into f609e31113424eafaebf8914ffd9d9df(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:37,356 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:37,356 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/A, priority=12, startTime=1731878736436; duration=0sec 2024-11-17T21:25:37,356 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:37,356 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:A 2024-11-17T21:25:37,443 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/8f95d7f5bc4245ebbe02faec13314310 2024-11-17T21:25:37,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/cc8ddcf24f354b99a34233aeb096b928 is 50, key is test_row_0/C:col10/1731878735909/Put/seqid=0 2024-11-17T21:25:37,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742182_1358 (size=12001) 2024-11-17T21:25:37,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878797780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:37,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:37,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878797791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:37,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:37,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878797801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:37,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878797801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878797801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:37,853 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/cc8ddcf24f354b99a34233aeb096b928 2024-11-17T21:25:37,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/c2187f376ddd4904ae35a41119d49522 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c2187f376ddd4904ae35a41119d49522 2024-11-17T21:25:37,868 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c2187f376ddd4904ae35a41119d49522, entries=150, sequenceid=90, filesize=30.2 K 2024-11-17T21:25:37,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/8f95d7f5bc4245ebbe02faec13314310 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/8f95d7f5bc4245ebbe02faec13314310 2024-11-17T21:25:37,873 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/8f95d7f5bc4245ebbe02faec13314310, entries=150, sequenceid=90, filesize=11.7 K 2024-11-17T21:25:37,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/cc8ddcf24f354b99a34233aeb096b928 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/cc8ddcf24f354b99a34233aeb096b928 2024-11-17T21:25:37,886 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/cc8ddcf24f354b99a34233aeb096b928, entries=150, sequenceid=90, filesize=11.7 K 2024-11-17T21:25:37,888 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c3ee12f1af122429c9b29c87eab606c8 in 1364ms, sequenceid=90, compaction requested=false 2024-11-17T21:25:37,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:37,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:37,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-17T21:25:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-17T21:25:37,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-17T21:25:37,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8340 sec 2024-11-17T21:25:37,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.8410 sec 2024-11-17T21:25:38,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T21:25:38,202 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:38,203 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:25:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-17T21:25:38,205 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:25:38,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T21:25:38,207 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:25:38,207 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:25:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T21:25:38,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-17T21:25:38,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:38,363 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T21:25:38,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b7a25c50dadd43a3bb1ba984a33a478c_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878736623/Put/seqid=0 2024-11-17T21:25:38,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742183_1359 (size=12154) 2024-11-17T21:25:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T21:25:38,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:38,786 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b7a25c50dadd43a3bb1ba984a33a478c_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b7a25c50dadd43a3bb1ba984a33a478c_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:38,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/229a739341324de8a362564c346a0460, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:38,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/229a739341324de8a362564c346a0460 is 175, key is test_row_0/A:col10/1731878736623/Put/seqid=0 2024-11-17T21:25:38,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742184_1360 (size=30955) 2024-11-17T21:25:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:38,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:38,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878798817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:38,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:38,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:38,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:38,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878798820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T21:25:38,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878798821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:38,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878798821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:38,832 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_2' in c3ee12f1af122429c9b29c87eab606c8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:38,832 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_0' in c3ee12f1af122429c9b29c87eab606c8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:38,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878798824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:38,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:38,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878798930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:38,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:38,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878798930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:38,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878798941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:39,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878799141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:39,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878799141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:39,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:39,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878799151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:39,192 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/229a739341324de8a362564c346a0460 2024-11-17T21:25:39,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/918ff5d630a442d5b78af4dbd29bad1e is 50, key is test_row_0/B:col10/1731878736623/Put/seqid=0 2024-11-17T21:25:39,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742185_1361 (size=12001) 2024-11-17T21:25:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T21:25:39,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878799451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:39,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878799451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:39,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878799461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:39,609 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/918ff5d630a442d5b78af4dbd29bad1e 2024-11-17T21:25:39,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/73e6509f1195408789136114d4cb4291 is 50, key is test_row_0/C:col10/1731878736623/Put/seqid=0 2024-11-17T21:25:39,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742186_1362 (size=12001) 2024-11-17T21:25:39,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878799971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:39,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:39,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878799971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:39,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:39,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878799981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:40,019 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/73e6509f1195408789136114d4cb4291 2024-11-17T21:25:40,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/229a739341324de8a362564c346a0460 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/229a739341324de8a362564c346a0460 2024-11-17T21:25:40,025 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/229a739341324de8a362564c346a0460, entries=150, sequenceid=117, filesize=30.2 K 2024-11-17T21:25:40,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/918ff5d630a442d5b78af4dbd29bad1e as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/918ff5d630a442d5b78af4dbd29bad1e 2024-11-17T21:25:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,029 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/918ff5d630a442d5b78af4dbd29bad1e, entries=150, sequenceid=117, filesize=11.7 K 2024-11-17T21:25:40,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/73e6509f1195408789136114d4cb4291 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/73e6509f1195408789136114d4cb4291 2024-11-17T21:25:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,035 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/73e6509f1195408789136114d4cb4291, entries=150, sequenceid=117, filesize=11.7 K 2024-11-17T21:25:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,036 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c3ee12f1af122429c9b29c87eab606c8 in 1673ms, sequenceid=117, compaction requested=true 2024-11-17T21:25:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:40,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-17T21:25:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-17T21:25:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-17T21:25:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8310 sec 2024-11-17T21:25:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8370 sec 2024-11-17T21:25:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T21:25:40,351 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,351 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:25:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-17T21:25:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T21:25:40,354 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:25:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,355 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:25:40,355 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:25:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T21:25:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-17T21:25:40,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:40,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,508 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:25:40,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179ce17fb8d3444600a57b80abbdc5c62a_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878738823/Put/seqid=0 2024-11-17T21:25:40,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742187_1363 (size=9714) 2024-11-17T21:25:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,555 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179ce17fb8d3444600a57b80abbdc5c62a_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179ce17fb8d3444600a57b80abbdc5c62a_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/f1bca9b5083f4fe9b4786b0fd885ae25, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/f1bca9b5083f4fe9b4786b0fd885ae25 is 175, key is test_row_0/A:col10/1731878738823/Put/seqid=0 2024-11-17T21:25:40,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742188_1364 (size=22361) 2024-11-17T21:25:40,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,578 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/f1bca9b5083f4fe9b4786b0fd885ae25 2024-11-17T21:25:40,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/ef8e07b98fe14e24b55e2efecbfae23f is 50, key is test_row_0/B:col10/1731878738823/Put/seqid=0 2024-11-17T21:25:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742189_1365 (size=9657) 2024-11-17T21:25:40,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T21:25:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T21:25:41,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878801013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878801013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878801023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878801025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878801026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,046 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/ef8e07b98fe14e24b55e2efecbfae23f 2024-11-17T21:25:41,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/ad0ab517ae0e4375b0022c23f106259d is 50, key is test_row_0/C:col10/1731878738823/Put/seqid=0 2024-11-17T21:25:41,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742190_1366 (size=9657) 2024-11-17T21:25:41,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878801131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878801131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878801141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878801141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878801141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878801351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878801351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878801351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878801351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878801351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,472 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/ad0ab517ae0e4375b0022c23f106259d 2024-11-17T21:25:41,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/f1bca9b5083f4fe9b4786b0fd885ae25 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/f1bca9b5083f4fe9b4786b0fd885ae25 2024-11-17T21:25:41,485 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/f1bca9b5083f4fe9b4786b0fd885ae25, entries=100, sequenceid=128, filesize=21.8 K 2024-11-17T21:25:41,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/ef8e07b98fe14e24b55e2efecbfae23f as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ef8e07b98fe14e24b55e2efecbfae23f 2024-11-17T21:25:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T21:25:41,492 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ef8e07b98fe14e24b55e2efecbfae23f, entries=100, sequenceid=128, filesize=9.4 K 2024-11-17T21:25:41,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/ad0ab517ae0e4375b0022c23f106259d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/ad0ab517ae0e4375b0022c23f106259d 2024-11-17T21:25:41,499 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/ad0ab517ae0e4375b0022c23f106259d, entries=100, sequenceid=128, filesize=9.4 K 2024-11-17T21:25:41,500 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c3ee12f1af122429c9b29c87eab606c8 in 992ms, sequenceid=128, compaction requested=true 2024-11-17T21:25:41,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:41,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:41,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-17T21:25:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-17T21:25:41,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-17T21:25:41,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1590 sec 2024-11-17T21:25:41,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.1650 sec 2024-11-17T21:25:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:41,675 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:41,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T21:25:41,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117684d08f5e64042a9b1c7b1191364cc9b_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878741022/Put/seqid=0 2024-11-17T21:25:41,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878801683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878801685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742191_1367 (size=14794) 2024-11-17T21:25:41,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878801689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,700 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:41,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,705 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117684d08f5e64042a9b1c7b1191364cc9b_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117684d08f5e64042a9b1c7b1191364cc9b_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:41,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878801691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,706 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/0af91247a6ee432a989cae4b2fed8154, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:41,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/0af91247a6ee432a989cae4b2fed8154 is 175, key is test_row_0/A:col10/1731878741022/Put/seqid=0 2024-11-17T21:25:41,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878801698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742192_1368 (size=39749) 2024-11-17T21:25:41,725 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/0af91247a6ee432a989cae4b2fed8154 2024-11-17T21:25:41,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/342099b99b7243a1ba188e82cc3d8cb3 is 50, key is test_row_0/B:col10/1731878741022/Put/seqid=0 2024-11-17T21:25:41,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742193_1369 (size=12151) 2024-11-17T21:25:41,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/342099b99b7243a1ba188e82cc3d8cb3 2024-11-17T21:25:41,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/5f7bb0be225a4c57aaf6a898b17ce3a0 is 50, key is test_row_0/C:col10/1731878741022/Put/seqid=0 2024-11-17T21:25:41,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878801803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878801803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878801811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878801812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878801821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:41,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742194_1370 (size=12151) 2024-11-17T21:25:41,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/5f7bb0be225a4c57aaf6a898b17ce3a0 2024-11-17T21:25:41,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/0af91247a6ee432a989cae4b2fed8154 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/0af91247a6ee432a989cae4b2fed8154 2024-11-17T21:25:41,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/0af91247a6ee432a989cae4b2fed8154, entries=200, sequenceid=154, filesize=38.8 K 2024-11-17T21:25:41,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/342099b99b7243a1ba188e82cc3d8cb3 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/342099b99b7243a1ba188e82cc3d8cb3 2024-11-17T21:25:41,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/342099b99b7243a1ba188e82cc3d8cb3, entries=150, sequenceid=154, filesize=11.9 K 2024-11-17T21:25:41,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/5f7bb0be225a4c57aaf6a898b17ce3a0 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/5f7bb0be225a4c57aaf6a898b17ce3a0 2024-11-17T21:25:41,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/5f7bb0be225a4c57aaf6a898b17ce3a0, entries=150, sequenceid=154, filesize=11.9 K 2024-11-17T21:25:41,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for c3ee12f1af122429c9b29c87eab606c8 in 203ms, sequenceid=154, compaction requested=true 2024-11-17T21:25:41,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:41,879 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T21:25:41,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:25:41,880 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 155113 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T21:25:41,880 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/A is initiating minor compaction (all files) 2024-11-17T21:25:41,881 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/A in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:41,881 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/f609e31113424eafaebf8914ffd9d9df, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c2187f376ddd4904ae35a41119d49522, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/229a739341324de8a362564c346a0460, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/f1bca9b5083f4fe9b4786b0fd885ae25, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/0af91247a6ee432a989cae4b2fed8154] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=151.5 K 2024-11-17T21:25:41,881 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:41,881 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/f609e31113424eafaebf8914ffd9d9df, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c2187f376ddd4904ae35a41119d49522, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/229a739341324de8a362564c346a0460, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/f1bca9b5083f4fe9b4786b0fd885ae25, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/0af91247a6ee432a989cae4b2fed8154] 2024-11-17T21:25:41,881 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting f609e31113424eafaebf8914ffd9d9df, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731878735870 2024-11-17T21:25:41,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:41,882 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T21:25:41,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:41,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:41,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:25:41,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:41,882 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2187f376ddd4904ae35a41119d49522, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731878735900 2024-11-17T21:25:41,883 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 229a739341324de8a362564c346a0460, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731878736611 2024-11-17T21:25:41,883 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1bca9b5083f4fe9b4786b0fd885ae25, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731878738819 2024-11-17T21:25:41,883 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0af91247a6ee432a989cae4b2fed8154, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731878741008 2024-11-17T21:25:41,884 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57949 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T21:25:41,884 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/B is initiating minor compaction (all files) 2024-11-17T21:25:41,884 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/B in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:41,884 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/2053d16cfb6349a7bdd6855764da2999, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/8f95d7f5bc4245ebbe02faec13314310, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/918ff5d630a442d5b78af4dbd29bad1e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ef8e07b98fe14e24b55e2efecbfae23f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/342099b99b7243a1ba188e82cc3d8cb3] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=56.6 K 2024-11-17T21:25:41,884 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 2053d16cfb6349a7bdd6855764da2999, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731878735870 2024-11-17T21:25:41,885 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f95d7f5bc4245ebbe02faec13314310, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731878735900 2024-11-17T21:25:41,885 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 918ff5d630a442d5b78af4dbd29bad1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731878736611 2024-11-17T21:25:41,887 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting ef8e07b98fe14e24b55e2efecbfae23f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731878738819 2024-11-17T21:25:41,887 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 342099b99b7243a1ba188e82cc3d8cb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731878741022 2024-11-17T21:25:41,905 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:41,914 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#B#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:41,914 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/d5a6095e5737413eba3881aa74bd6971 is 50, key is test_row_0/B:col10/1731878741022/Put/seqid=0 2024-11-17T21:25:41,917 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117d48a75776f6446159be14fedd9a3a1ec_c3ee12f1af122429c9b29c87eab606c8 store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:41,920 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117d48a75776f6446159be14fedd9a3a1ec_c3ee12f1af122429c9b29c87eab606c8, store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:41,920 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d48a75776f6446159be14fedd9a3a1ec_c3ee12f1af122429c9b29c87eab606c8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:41,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742195_1371 (size=12459) 2024-11-17T21:25:41,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742196_1372 (size=4469) 2024-11-17T21:25:41,992 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#A#compaction#314 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:41,992 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/b4e4b4a6458b4eec955394aad1fb6844 is 175, key is test_row_0/A:col10/1731878741022/Put/seqid=0 2024-11-17T21:25:42,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742197_1373 (size=31413) 2024-11-17T21:25:42,023 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/b4e4b4a6458b4eec955394aad1fb6844 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/b4e4b4a6458b4eec955394aad1fb6844 2024-11-17T21:25:42,046 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/A of c3ee12f1af122429c9b29c87eab606c8 into b4e4b4a6458b4eec955394aad1fb6844(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:42,046 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:42,046 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/A, priority=11, startTime=1731878741878; duration=0sec 2024-11-17T21:25:42,046 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:42,046 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:A 2024-11-17T21:25:42,046 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T21:25:42,048 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57949 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T21:25:42,048 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/C is initiating minor compaction (all files) 2024-11-17T21:25:42,048 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/C in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:42,048 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/f46fa0e3d8ee469b87208b9aa85bb9af, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/cc8ddcf24f354b99a34233aeb096b928, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/73e6509f1195408789136114d4cb4291, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/ad0ab517ae0e4375b0022c23f106259d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/5f7bb0be225a4c57aaf6a898b17ce3a0] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=56.6 K 2024-11-17T21:25:42,048 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting f46fa0e3d8ee469b87208b9aa85bb9af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731878735870 2024-11-17T21:25:42,049 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc8ddcf24f354b99a34233aeb096b928, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731878735900 2024-11-17T21:25:42,049 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73e6509f1195408789136114d4cb4291, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731878736611 2024-11-17T21:25:42,049 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad0ab517ae0e4375b0022c23f106259d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731878738819 2024-11-17T21:25:42,050 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f7bb0be225a4c57aaf6a898b17ce3a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731878741022 2024-11-17T21:25:42,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:42,053 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:42,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T21:25:42,079 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#C#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:42,080 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/8f83bcc9394d46ca93c1a6d17cbaf065 is 50, key is test_row_0/C:col10/1731878741022/Put/seqid=0 2024-11-17T21:25:42,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172c724278cb6f4379a4f2a49606165e61_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878741696/Put/seqid=0 2024-11-17T21:25:42,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742198_1374 (size=12459) 2024-11-17T21:25:42,142 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/8f83bcc9394d46ca93c1a6d17cbaf065 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/8f83bcc9394d46ca93c1a6d17cbaf065 2024-11-17T21:25:42,147 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/C of c3ee12f1af122429c9b29c87eab606c8 into 8f83bcc9394d46ca93c1a6d17cbaf065(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:42,147 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:42,149 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/C, priority=11, startTime=1731878741882; duration=0sec 2024-11-17T21:25:42,149 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:42,149 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:C 2024-11-17T21:25:42,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742199_1375 (size=17284) 2024-11-17T21:25:42,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878802127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878802142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878802144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878802156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878802156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878802262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878802262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878802271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878802281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878802281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,370 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/d5a6095e5737413eba3881aa74bd6971 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d5a6095e5737413eba3881aa74bd6971 2024-11-17T21:25:42,378 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/B of c3ee12f1af122429c9b29c87eab606c8 into d5a6095e5737413eba3881aa74bd6971(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:42,378 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:42,378 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/B, priority=11, startTime=1731878741881; duration=0sec 2024-11-17T21:25:42,378 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:42,378 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:B 2024-11-17T21:25:42,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878802481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878802481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878802481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878802493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878802493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T21:25:42,501 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:42,503 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:25:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-17T21:25:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T21:25:42,505 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:25:42,505 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:25:42,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:25:42,558 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:42,573 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172c724278cb6f4379a4f2a49606165e61_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172c724278cb6f4379a4f2a49606165e61_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:42,574 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/06a968e631374cb8991575ed8dbb27c9, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:42,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/06a968e631374cb8991575ed8dbb27c9 is 175, key is test_row_0/A:col10/1731878741696/Put/seqid=0 2024-11-17T21:25:42,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742200_1376 (size=48389) 2024-11-17T21:25:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T21:25:42,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T21:25:42,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:42,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:42,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:42,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:42,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:42,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878802802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878802803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878802803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878802803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878802803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:42,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T21:25:42,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:42,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:42,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:42,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T21:25:42,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T21:25:42,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:42,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:42,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:42,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:42,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:42,993 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/06a968e631374cb8991575ed8dbb27c9 2024-11-17T21:25:43,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/7f2647ad91794996b8721fdba233c9ec is 50, key is test_row_0/B:col10/1731878741696/Put/seqid=0 2024-11-17T21:25:43,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742201_1377 (size=12151) 2024-11-17T21:25:43,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T21:25:43,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:43,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:43,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:43,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:43,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T21:25:43,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T21:25:43,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:43,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:43,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:43,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:43,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:43,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878803322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:43,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:43,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:43,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:43,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878803322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:43,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:43,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:43,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:43,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878803322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:43,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878803322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:43,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:43,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878803322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:43,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:43,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:43,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:43,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/7f2647ad91794996b8721fdba233c9ec 2024-11-17T21:25:43,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T21:25:43,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:43,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:43,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:43,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:43,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:43,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/751619eb77dd4053950d4bfb8b5996a0 is 50, key is test_row_0/C:col10/1731878741696/Put/seqid=0 2024-11-17T21:25:43,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742202_1378 (size=12151) 2024-11-17T21:25:43,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/751619eb77dd4053950d4bfb8b5996a0 2024-11-17T21:25:43,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/06a968e631374cb8991575ed8dbb27c9 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/06a968e631374cb8991575ed8dbb27c9 2024-11-17T21:25:43,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/06a968e631374cb8991575ed8dbb27c9, entries=250, sequenceid=167, filesize=47.3 K 2024-11-17T21:25:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/7f2647ad91794996b8721fdba233c9ec as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/7f2647ad91794996b8721fdba233c9ec 2024-11-17T21:25:43,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/7f2647ad91794996b8721fdba233c9ec, entries=150, sequenceid=167, filesize=11.9 K 2024-11-17T21:25:43,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/751619eb77dd4053950d4bfb8b5996a0 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/751619eb77dd4053950d4bfb8b5996a0 2024-11-17T21:25:43,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/751619eb77dd4053950d4bfb8b5996a0, entries=150, sequenceid=167, filesize=11.9 K 2024-11-17T21:25:43,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for c3ee12f1af122429c9b29c87eab606c8 in 1447ms, sequenceid=167, compaction requested=false 2024-11-17T21:25:43,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:43,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T21:25:43,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:43,582 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T21:25:43,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f628693fd5244c42a4d7f61ed0b763b3_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878742121/Put/seqid=0 2024-11-17T21:25:43,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742203_1379 (size=12304) 2024-11-17T21:25:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T21:25:44,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:44,030 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f628693fd5244c42a4d7f61ed0b763b3_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f628693fd5244c42a4d7f61ed0b763b3_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:44,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/2a0ecdeada3e4c11ae266f15d9ab0858, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:44,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/2a0ecdeada3e4c11ae266f15d9ab0858 is 175, key is test_row_0/A:col10/1731878742121/Put/seqid=0 2024-11-17T21:25:44,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742204_1380 (size=31105) 2024-11-17T21:25:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:44,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:44,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878804378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:44,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878804380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:44,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:44,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878804394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:44,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878804397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:44,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878804405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:44,444 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/2a0ecdeada3e4c11ae266f15d9ab0858 2024-11-17T21:25:44,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/0fca2501198a4393bf6db71007edd43a is 50, key is test_row_0/B:col10/1731878742121/Put/seqid=0 2024-11-17T21:25:44,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742205_1381 (size=12151) 2024-11-17T21:25:44,506 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/0fca2501198a4393bf6db71007edd43a 2024-11-17T21:25:44,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878804501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878804501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:44,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/c83a614fb8e14db39abd70a599336bbf is 50, key is test_row_0/C:col10/1731878742121/Put/seqid=0 2024-11-17T21:25:44,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878804521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:44,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:44,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878804531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:44,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878804532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:44,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742206_1382 (size=12151) 2024-11-17T21:25:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T21:25:44,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:44,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878804722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:44,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:44,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878804722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:44,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878804741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:44,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878804751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:44,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878804751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:44,960 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/c83a614fb8e14db39abd70a599336bbf 2024-11-17T21:25:44,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/2a0ecdeada3e4c11ae266f15d9ab0858 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2a0ecdeada3e4c11ae266f15d9ab0858 2024-11-17T21:25:44,969 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2a0ecdeada3e4c11ae266f15d9ab0858, entries=150, sequenceid=194, filesize=30.4 K 2024-11-17T21:25:44,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/0fca2501198a4393bf6db71007edd43a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/0fca2501198a4393bf6db71007edd43a 2024-11-17T21:25:44,973 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/0fca2501198a4393bf6db71007edd43a, entries=150, sequenceid=194, filesize=11.9 K 2024-11-17T21:25:44,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/c83a614fb8e14db39abd70a599336bbf as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/c83a614fb8e14db39abd70a599336bbf 2024-11-17T21:25:44,982 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/c83a614fb8e14db39abd70a599336bbf, entries=150, sequenceid=194, filesize=11.9 K 2024-11-17T21:25:44,983 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c3ee12f1af122429c9b29c87eab606c8 in 1401ms, sequenceid=194, compaction requested=true 2024-11-17T21:25:44,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:44,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:44,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-17T21:25:44,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-17T21:25:44,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-17T21:25:44,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4790 sec 2024-11-17T21:25:44,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.4850 sec 2024-11-17T21:25:45,050 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:45,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T21:25:45,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:45,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a6e725b484514fbcb15f8ba583acd639_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878744391/Put/seqid=0 2024-11-17T21:25:45,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742207_1383 (size=14794) 2024-11-17T21:25:45,109 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:45,118 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a6e725b484514fbcb15f8ba583acd639_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a6e725b484514fbcb15f8ba583acd639_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:45,119 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/14acceec788a471ea255ae9d8729cc7a, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:45,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/14acceec788a471ea255ae9d8729cc7a is 175, key is test_row_0/A:col10/1731878744391/Put/seqid=0 2024-11-17T21:25:45,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878805112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878805112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878805113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878805115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878805115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742208_1384 (size=39749) 2024-11-17T21:25:45,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878805231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878805232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878805232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878805242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878805242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878805451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878805451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878805451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878805462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878805463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,534 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/14acceec788a471ea255ae9d8729cc7a 2024-11-17T21:25:45,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/ad393b446b334f9d894cf4e272131024 is 50, key is test_row_0/B:col10/1731878744391/Put/seqid=0 2024-11-17T21:25:45,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742209_1385 (size=12151) 2024-11-17T21:25:45,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878805771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878805771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878805771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878805772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:45,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878805791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:45,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:46,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/ad393b446b334f9d894cf4e272131024 2024-11-17T21:25:46,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/d1e8fedb176142ff8edcdfba2bfecc2b is 50, key is test_row_0/C:col10/1731878744391/Put/seqid=0 2024-11-17T21:25:46,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742210_1386 (size=12151) 2024-11-17T21:25:46,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878806291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:46,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878806291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:46,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878806292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:46,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878806292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:46,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878806321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:46,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/d1e8fedb176142ff8edcdfba2bfecc2b 2024-11-17T21:25:46,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/14acceec788a471ea255ae9d8729cc7a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/14acceec788a471ea255ae9d8729cc7a 2024-11-17T21:25:46,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/14acceec788a471ea255ae9d8729cc7a, entries=200, sequenceid=206, filesize=38.8 K 2024-11-17T21:25:46,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/ad393b446b334f9d894cf4e272131024 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ad393b446b334f9d894cf4e272131024 2024-11-17T21:25:46,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ad393b446b334f9d894cf4e272131024, entries=150, sequenceid=206, filesize=11.9 K 2024-11-17T21:25:46,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/d1e8fedb176142ff8edcdfba2bfecc2b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d1e8fedb176142ff8edcdfba2bfecc2b 2024-11-17T21:25:46,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d1e8fedb176142ff8edcdfba2bfecc2b, entries=150, sequenceid=206, filesize=11.9 K 2024-11-17T21:25:46,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c3ee12f1af122429c9b29c87eab606c8 in 1472ms, sequenceid=206, compaction requested=true 2024-11-17T21:25:46,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:46,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:25:46,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:46,523 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:46,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:46,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:46,523 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:46,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:25:46,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:46,524 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:46,524 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150656 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:46,524 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/B is initiating minor compaction (all files) 2024-11-17T21:25:46,524 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/A is initiating minor compaction (all files) 2024-11-17T21:25:46,524 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/B in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:46,524 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/A in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:46,525 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d5a6095e5737413eba3881aa74bd6971, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/7f2647ad91794996b8721fdba233c9ec, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/0fca2501198a4393bf6db71007edd43a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ad393b446b334f9d894cf4e272131024] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=47.8 K 2024-11-17T21:25:46,525 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/b4e4b4a6458b4eec955394aad1fb6844, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/06a968e631374cb8991575ed8dbb27c9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2a0ecdeada3e4c11ae266f15d9ab0858, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/14acceec788a471ea255ae9d8729cc7a] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=147.1 K 2024-11-17T21:25:46,525 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:46,525 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/b4e4b4a6458b4eec955394aad1fb6844, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/06a968e631374cb8991575ed8dbb27c9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2a0ecdeada3e4c11ae266f15d9ab0858, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/14acceec788a471ea255ae9d8729cc7a] 2024-11-17T21:25:46,525 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d5a6095e5737413eba3881aa74bd6971, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731878741022 2024-11-17T21:25:46,525 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4e4b4a6458b4eec955394aad1fb6844, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731878741022 2024-11-17T21:25:46,525 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f2647ad91794996b8721fdba233c9ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731878741685 2024-11-17T21:25:46,525 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06a968e631374cb8991575ed8dbb27c9, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731878741681 2024-11-17T21:25:46,526 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a0ecdeada3e4c11ae266f15d9ab0858, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731878742121 2024-11-17T21:25:46,526 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fca2501198a4393bf6db71007edd43a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731878742121 2024-11-17T21:25:46,526 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14acceec788a471ea255ae9d8729cc7a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731878744376 2024-11-17T21:25:46,526 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting ad393b446b334f9d894cf4e272131024, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731878744376 2024-11-17T21:25:46,545 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:46,562 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#B#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:46,562 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/77c9a38585fe4107af205146f8583589 is 50, key is test_row_0/B:col10/1731878744391/Put/seqid=0 2024-11-17T21:25:46,569 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117e79b9a0a7f9f4497b37186e263f47cd6_c3ee12f1af122429c9b29c87eab606c8 store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:46,571 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117e79b9a0a7f9f4497b37186e263f47cd6_c3ee12f1af122429c9b29c87eab606c8, store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:46,572 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e79b9a0a7f9f4497b37186e263f47cd6_c3ee12f1af122429c9b29c87eab606c8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:46,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742212_1388 (size=4469) 2024-11-17T21:25:46,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742211_1387 (size=12595) 2024-11-17T21:25:46,634 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#A#compaction#326 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:46,635 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/c8d38c893a0e45429e835e367ba0fa4c is 175, key is test_row_0/A:col10/1731878744391/Put/seqid=0 2024-11-17T21:25:46,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742213_1389 (size=31549) 2024-11-17T21:25:46,662 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/c8d38c893a0e45429e835e367ba0fa4c as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c8d38c893a0e45429e835e367ba0fa4c 2024-11-17T21:25:46,667 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/A of c3ee12f1af122429c9b29c87eab606c8 into c8d38c893a0e45429e835e367ba0fa4c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:46,667 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:46,667 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/A, priority=12, startTime=1731878746523; duration=0sec 2024-11-17T21:25:46,667 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:46,667 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:A 2024-11-17T21:25:46,667 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:46,669 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:46,669 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/C is initiating minor compaction (all files) 2024-11-17T21:25:46,669 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/C in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:46,669 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/8f83bcc9394d46ca93c1a6d17cbaf065, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/751619eb77dd4053950d4bfb8b5996a0, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/c83a614fb8e14db39abd70a599336bbf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d1e8fedb176142ff8edcdfba2bfecc2b] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=47.8 K 2024-11-17T21:25:46,669 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f83bcc9394d46ca93c1a6d17cbaf065, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731878741022 2024-11-17T21:25:46,670 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 751619eb77dd4053950d4bfb8b5996a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731878741685 2024-11-17T21:25:46,670 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting c83a614fb8e14db39abd70a599336bbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731878742121 2024-11-17T21:25:46,670 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1e8fedb176142ff8edcdfba2bfecc2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731878744376 2024-11-17T21:25:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T21:25:46,671 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:46,671 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:25:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-17T21:25:46,674 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:25:46,676 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:25:46,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:25:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T21:25:46,681 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#C#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:46,682 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/348c6236986e4d98bbb9082a2e2cb8e7 is 50, key is test_row_0/C:col10/1731878744391/Put/seqid=0 2024-11-17T21:25:46,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742214_1390 (size=12595) 2024-11-17T21:25:46,699 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/348c6236986e4d98bbb9082a2e2cb8e7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/348c6236986e4d98bbb9082a2e2cb8e7 2024-11-17T21:25:46,703 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/C of c3ee12f1af122429c9b29c87eab606c8 into 348c6236986e4d98bbb9082a2e2cb8e7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:46,703 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:46,703 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/C, priority=12, startTime=1731878746523; duration=0sec 2024-11-17T21:25:46,703 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:46,703 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:C 2024-11-17T21:25:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T21:25:46,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T21:25:46,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:46,832 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T21:25:46,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117715728ca199f452dbf9fedb79fd6d1f8_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878745110/Put/seqid=0 2024-11-17T21:25:46,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742215_1391 (size=12304) 2024-11-17T21:25:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:46,866 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117715728ca199f452dbf9fedb79fd6d1f8_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117715728ca199f452dbf9fedb79fd6d1f8_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:46,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/d20da1981dad4abeb6ce42fd382a0f2b, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:46,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/d20da1981dad4abeb6ce42fd382a0f2b is 175, key is test_row_0/A:col10/1731878745110/Put/seqid=0 2024-11-17T21:25:46,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742216_1392 (size=31105) 2024-11-17T21:25:46,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T21:25:47,053 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/77c9a38585fe4107af205146f8583589 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/77c9a38585fe4107af205146f8583589 2024-11-17T21:25:47,058 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/B of c3ee12f1af122429c9b29c87eab606c8 into 77c9a38585fe4107af205146f8583589(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:47,058 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:47,059 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/B, priority=12, startTime=1731878746523; duration=0sec 2024-11-17T21:25:47,059 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:47,059 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:B 2024-11-17T21:25:47,283 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/d20da1981dad4abeb6ce42fd382a0f2b 2024-11-17T21:25:47,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/d923995d2d4a4a4f9895f6f54c2db9e5 is 50, key is test_row_0/B:col10/1731878745110/Put/seqid=0 2024-11-17T21:25:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T21:25:47,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742217_1393 (size=12151) 2024-11-17T21:25:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:47,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:47,314 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/d923995d2d4a4a4f9895f6f54c2db9e5 2024-11-17T21:25:47,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/32a72f6d08024f3ba6418773e1b1e0bc is 50, key is test_row_0/C:col10/1731878745110/Put/seqid=0 2024-11-17T21:25:47,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878807330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742218_1394 (size=12151) 2024-11-17T21:25:47,350 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/32a72f6d08024f3ba6418773e1b1e0bc 2024-11-17T21:25:47,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878807333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878807335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878807336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/d20da1981dad4abeb6ce42fd382a0f2b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/d20da1981dad4abeb6ce42fd382a0f2b 2024-11-17T21:25:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878807351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,361 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_0' in c3ee12f1af122429c9b29c87eab606c8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,365 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/d20da1981dad4abeb6ce42fd382a0f2b, entries=150, sequenceid=232, filesize=30.4 K 2024-11-17T21:25:47,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/d923995d2d4a4a4f9895f6f54c2db9e5 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d923995d2d4a4a4f9895f6f54c2db9e5 2024-11-17T21:25:47,376 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d923995d2d4a4a4f9895f6f54c2db9e5, entries=150, sequenceid=232, filesize=11.9 K 2024-11-17T21:25:47,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/32a72f6d08024f3ba6418773e1b1e0bc as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/32a72f6d08024f3ba6418773e1b1e0bc 2024-11-17T21:25:47,384 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/32a72f6d08024f3ba6418773e1b1e0bc, entries=150, sequenceid=232, filesize=11.9 K 2024-11-17T21:25:47,386 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for c3ee12f1af122429c9b29c87eab606c8 in 555ms, sequenceid=232, compaction requested=false 2024-11-17T21:25:47,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:47,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:47,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-17T21:25:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-17T21:25:47,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-17T21:25:47,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 711 msec 2024-11-17T21:25:47,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 720 msec 2024-11-17T21:25:47,467 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:47,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T21:25:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:47,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a551ba985e9b45ef9ce20aa22ffadc56_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878747330/Put/seqid=0 2024-11-17T21:25:47,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742219_1395 (size=12304) 2024-11-17T21:25:47,514 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:47,523 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a551ba985e9b45ef9ce20aa22ffadc56_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a551ba985e9b45ef9ce20aa22ffadc56_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:47,524 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/0554d45f61074c7dbc97e5248fa30d7a, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:47,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/0554d45f61074c7dbc97e5248fa30d7a is 175, key is test_row_0/A:col10/1731878747330/Put/seqid=0 2024-11-17T21:25:47,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742220_1396 (size=31105) 2024-11-17T21:25:47,533 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/0554d45f61074c7dbc97e5248fa30d7a 2024-11-17T21:25:47,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/23431f91ce1341679c0659a32e015a52 is 50, key is test_row_0/B:col10/1731878747330/Put/seqid=0 2024-11-17T21:25:47,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878807537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742221_1397 (size=12151) 2024-11-17T21:25:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878807537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878807552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878807552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878807661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878807661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878807671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878807671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T21:25:47,812 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:47,813 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:25:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-17T21:25:47,815 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:25:47,816 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:25:47,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:25:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T21:25:47,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878807871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878807871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878807894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:47,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878807894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:47,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:47,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T21:25:47,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/23431f91ce1341679c0659a32e015a52 2024-11-17T21:25:47,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/d33469ea9bc8401d9f0e48db3ae6b6f4 is 50, key is test_row_0/C:col10/1731878747330/Put/seqid=0 2024-11-17T21:25:47,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T21:25:47,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:47,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:47,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:47,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:47,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:48,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742222_1398 (size=12151) 2024-11-17T21:25:48,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/d33469ea9bc8401d9f0e48db3ae6b6f4 2024-11-17T21:25:48,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/0554d45f61074c7dbc97e5248fa30d7a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/0554d45f61074c7dbc97e5248fa30d7a 2024-11-17T21:25:48,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/0554d45f61074c7dbc97e5248fa30d7a, entries=150, sequenceid=246, filesize=30.4 K 2024-11-17T21:25:48,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/23431f91ce1341679c0659a32e015a52 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/23431f91ce1341679c0659a32e015a52 2024-11-17T21:25:48,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/23431f91ce1341679c0659a32e015a52, entries=150, sequenceid=246, filesize=11.9 K 2024-11-17T21:25:48,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/d33469ea9bc8401d9f0e48db3ae6b6f4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d33469ea9bc8401d9f0e48db3ae6b6f4 2024-11-17T21:25:48,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d33469ea9bc8401d9f0e48db3ae6b6f4, entries=150, sequenceid=246, filesize=11.9 K 2024-11-17T21:25:48,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for c3ee12f1af122429c9b29c87eab606c8 in 571ms, sequenceid=246, compaction requested=true 2024-11-17T21:25:48,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:48,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:25:48,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:48,038 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:48,038 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:48,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:48,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:48,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:25:48,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:48,039 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:48,039 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/A is initiating minor compaction (all files) 2024-11-17T21:25:48,039 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:48,039 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/A in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:48,039 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/B is initiating minor compaction (all files) 2024-11-17T21:25:48,039 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c8d38c893a0e45429e835e367ba0fa4c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/d20da1981dad4abeb6ce42fd382a0f2b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/0554d45f61074c7dbc97e5248fa30d7a] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=91.6 K 2024-11-17T21:25:48,039 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/B in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:48,039 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:48,039 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/77c9a38585fe4107af205146f8583589, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d923995d2d4a4a4f9895f6f54c2db9e5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/23431f91ce1341679c0659a32e015a52] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=36.0 K 2024-11-17T21:25:48,039 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c8d38c893a0e45429e835e367ba0fa4c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/d20da1981dad4abeb6ce42fd382a0f2b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/0554d45f61074c7dbc97e5248fa30d7a] 2024-11-17T21:25:48,040 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8d38c893a0e45429e835e367ba0fa4c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731878744376 2024-11-17T21:25:48,040 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 77c9a38585fe4107af205146f8583589, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731878744376 2024-11-17T21:25:48,040 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting d20da1981dad4abeb6ce42fd382a0f2b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731878745110 2024-11-17T21:25:48,040 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d923995d2d4a4a4f9895f6f54c2db9e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731878745110 2024-11-17T21:25:48,040 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 23431f91ce1341679c0659a32e015a52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731878747330 2024-11-17T21:25:48,041 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0554d45f61074c7dbc97e5248fa30d7a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731878747330 2024-11-17T21:25:48,058 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:48,059 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#B#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:48,060 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/6e0f8df1af7248a08f01c20e17dc6a23 is 50, key is test_row_0/B:col10/1731878747330/Put/seqid=0 2024-11-17T21:25:48,064 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117c320a052a8b34cc79557b794a255251d_c3ee12f1af122429c9b29c87eab606c8 store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:48,066 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117c320a052a8b34cc79557b794a255251d_c3ee12f1af122429c9b29c87eab606c8, store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:48,066 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c320a052a8b34cc79557b794a255251d_c3ee12f1af122429c9b29c87eab606c8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:48,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742224_1400 (size=4469) 2024-11-17T21:25:48,092 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#A#compaction#336 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:48,092 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/9614ed06981c4bc39e9be92af5efad51 is 175, key is test_row_0/A:col10/1731878747330/Put/seqid=0 2024-11-17T21:25:48,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742223_1399 (size=12697) 2024-11-17T21:25:48,107 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/6e0f8df1af7248a08f01c20e17dc6a23 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/6e0f8df1af7248a08f01c20e17dc6a23 2024-11-17T21:25:48,113 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/B of c3ee12f1af122429c9b29c87eab606c8 into 6e0f8df1af7248a08f01c20e17dc6a23(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:48,113 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:48,113 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/B, priority=13, startTime=1731878748038; duration=0sec 2024-11-17T21:25:48,113 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:48,113 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:B 2024-11-17T21:25:48,113 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:48,117 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:48,117 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/C is initiating minor compaction (all files) 2024-11-17T21:25:48,117 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/C in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:48,117 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/348c6236986e4d98bbb9082a2e2cb8e7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/32a72f6d08024f3ba6418773e1b1e0bc, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d33469ea9bc8401d9f0e48db3ae6b6f4] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=36.0 K 2024-11-17T21:25:48,117 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 348c6236986e4d98bbb9082a2e2cb8e7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731878744376 2024-11-17T21:25:48,118 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 32a72f6d08024f3ba6418773e1b1e0bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731878745110 2024-11-17T21:25:48,118 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d33469ea9bc8401d9f0e48db3ae6b6f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731878747330 2024-11-17T21:25:48,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742225_1401 (size=31651) 2024-11-17T21:25:48,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T21:25:48,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:48,123 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T21:25:48,131 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#C#compaction#337 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:48,132 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/d73219b8b9a84d6d871e0b76cfad9f76 is 50, key is test_row_0/C:col10/1731878747330/Put/seqid=0 2024-11-17T21:25:48,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117339e0b23760e4630a3889205fe36c55e_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878747535/Put/seqid=0 2024-11-17T21:25:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T21:25:48,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742227_1403 (size=12454) 2024-11-17T21:25:48,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:48,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742226_1402 (size=12697) 2024-11-17T21:25:48,165 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117339e0b23760e4630a3889205fe36c55e_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117339e0b23760e4630a3889205fe36c55e_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:48,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/1bb0fb38efb34301aea9c962cfab4972, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:48,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/1bb0fb38efb34301aea9c962cfab4972 is 175, key is test_row_0/A:col10/1731878747535/Put/seqid=0 2024-11-17T21:25:48,178 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/d73219b8b9a84d6d871e0b76cfad9f76 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d73219b8b9a84d6d871e0b76cfad9f76 2024-11-17T21:25:48,184 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/C of c3ee12f1af122429c9b29c87eab606c8 into d73219b8b9a84d6d871e0b76cfad9f76(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:48,184 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:48,184 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/C, priority=13, startTime=1731878748038; duration=0sec 2024-11-17T21:25:48,184 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:48,184 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:C 2024-11-17T21:25:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:48,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:48,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742228_1404 (size=31255) 2024-11-17T21:25:48,196 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/1bb0fb38efb34301aea9c962cfab4972 2024-11-17T21:25:48,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/5574695a425d47c7a8c527594164fa54 is 50, key is test_row_0/B:col10/1731878747535/Put/seqid=0 2024-11-17T21:25:48,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742229_1405 (size=12301) 2024-11-17T21:25:48,223 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/5574695a425d47c7a8c527594164fa54 2024-11-17T21:25:48,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/0e167a519dfc45aebc3d0378d00f4a66 is 50, key is test_row_0/C:col10/1731878747535/Put/seqid=0 2024-11-17T21:25:48,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878808231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:48,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:48,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878808233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:48,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:48,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878808233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:48,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878808233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:48,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742230_1406 (size=12301) 2024-11-17T21:25:48,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878808352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:48,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878808352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:48,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878808352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:48,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878808354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T21:25:48,525 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/9614ed06981c4bc39e9be92af5efad51 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/9614ed06981c4bc39e9be92af5efad51 2024-11-17T21:25:48,530 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/A of c3ee12f1af122429c9b29c87eab606c8 into 9614ed06981c4bc39e9be92af5efad51(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:48,530 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:48,530 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/A, priority=13, startTime=1731878748038; duration=0sec 2024-11-17T21:25:48,530 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:48,530 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:A 2024-11-17T21:25:48,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878808561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878808561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:48,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878808561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:48,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878808571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:48,660 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/0e167a519dfc45aebc3d0378d00f4a66 2024-11-17T21:25:48,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/1bb0fb38efb34301aea9c962cfab4972 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/1bb0fb38efb34301aea9c962cfab4972 2024-11-17T21:25:48,669 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/1bb0fb38efb34301aea9c962cfab4972, entries=150, sequenceid=271, filesize=30.5 K 2024-11-17T21:25:48,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/5574695a425d47c7a8c527594164fa54 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/5574695a425d47c7a8c527594164fa54 2024-11-17T21:25:48,681 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/5574695a425d47c7a8c527594164fa54, entries=150, sequenceid=271, filesize=12.0 K 2024-11-17T21:25:48,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/0e167a519dfc45aebc3d0378d00f4a66 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/0e167a519dfc45aebc3d0378d00f4a66 2024-11-17T21:25:48,689 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/0e167a519dfc45aebc3d0378d00f4a66, entries=150, sequenceid=271, filesize=12.0 K 2024-11-17T21:25:48,690 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for c3ee12f1af122429c9b29c87eab606c8 in 567ms, sequenceid=271, compaction requested=false 2024-11-17T21:25:48,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:48,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:48,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-17T21:25:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-17T21:25:48,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-17T21:25:48,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 876 msec 2024-11-17T21:25:48,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 882 msec 2024-11-17T21:25:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:48,890 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:48,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T21:25:48,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111743f83c525c6441a68c9f08b8733c640e_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878748888/Put/seqid=0 2024-11-17T21:25:48,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742231_1407 (size=17534) 2024-11-17T21:25:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T21:25:48,951 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:48,951 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:25:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-17T21:25:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T21:25:48,953 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:25:48,953 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:25:48,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:25:48,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878808961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:48,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:48,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878808961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:48,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:48,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:48,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878808961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878808962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T21:25:49,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878809081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:49,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878809082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878809087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:49,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:49,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878809089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:49,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T21:25:49,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:49,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:49,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:49,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:49,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:49,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T21:25:49,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:49,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:49,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:49,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:49,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:49,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T21:25:49,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878809301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878809301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878809301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:49,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878809301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:49,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:49,306 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:49,310 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111743f83c525c6441a68c9f08b8733c640e_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111743f83c525c6441a68c9f08b8733c640e_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:49,310 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/1e2794d01ac64d74ab4493f2fe3c0428, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:49,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/1e2794d01ac64d74ab4493f2fe3c0428 is 175, key is test_row_0/A:col10/1731878748888/Put/seqid=0 2024-11-17T21:25:49,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742232_1408 (size=48639) 2024-11-17T21:25:49,335 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/1e2794d01ac64d74ab4493f2fe3c0428 2024-11-17T21:25:49,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/b9fc1757c9b74128ac8776c4d1c49d18 is 50, key is test_row_0/B:col10/1731878748888/Put/seqid=0 2024-11-17T21:25:49,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878809370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:49,382 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_0' in c3ee12f1af122429c9b29c87eab606c8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4267 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:49,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742233_1409 (size=12301) 2024-11-17T21:25:49,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T21:25:49,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:49,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:49,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:49,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:49,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:49,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:49,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T21:25:49,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:49,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:49,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:49,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:49,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T21:25:49,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878809611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:49,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878809611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:49,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878809611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:49,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878809611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:49,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T21:25:49,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:49,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:49,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:49,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:49,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:25:49,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/b9fc1757c9b74128ac8776c4d1c49d18 2024-11-17T21:25:49,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/2b7182d0d82e4a61b7ac9eae31f2e2d2 is 50, key is test_row_0/C:col10/1731878748888/Put/seqid=0 2024-11-17T21:25:49,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742234_1410 (size=12301) 2024-11-17T21:25:49,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/2b7182d0d82e4a61b7ac9eae31f2e2d2 2024-11-17T21:25:49,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/1e2794d01ac64d74ab4493f2fe3c0428 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/1e2794d01ac64d74ab4493f2fe3c0428 2024-11-17T21:25:49,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/1e2794d01ac64d74ab4493f2fe3c0428, entries=250, sequenceid=289, filesize=47.5 K 2024-11-17T21:25:49,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/b9fc1757c9b74128ac8776c4d1c49d18 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/b9fc1757c9b74128ac8776c4d1c49d18 2024-11-17T21:25:49,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/b9fc1757c9b74128ac8776c4d1c49d18, entries=150, sequenceid=289, filesize=12.0 K 2024-11-17T21:25:49,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/2b7182d0d82e4a61b7ac9eae31f2e2d2 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/2b7182d0d82e4a61b7ac9eae31f2e2d2 2024-11-17T21:25:49,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/2b7182d0d82e4a61b7ac9eae31f2e2d2, entries=150, sequenceid=289, filesize=12.0 K 2024-11-17T21:25:49,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for c3ee12f1af122429c9b29c87eab606c8 in 952ms, sequenceid=289, compaction requested=true 2024-11-17T21:25:49,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:49,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:25:49,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:49,843 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:49,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:49,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:49,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:25:49,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T21:25:49,844 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111545 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:49,844 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/A is initiating minor compaction (all files) 2024-11-17T21:25:49,844 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/A in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:49,844 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/9614ed06981c4bc39e9be92af5efad51, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/1bb0fb38efb34301aea9c962cfab4972, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/1e2794d01ac64d74ab4493f2fe3c0428] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=108.9 K 2024-11-17T21:25:49,844 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:49,844 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/9614ed06981c4bc39e9be92af5efad51, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/1bb0fb38efb34301aea9c962cfab4972, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/1e2794d01ac64d74ab4493f2fe3c0428] 2024-11-17T21:25:49,845 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9614ed06981c4bc39e9be92af5efad51, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731878747330 2024-11-17T21:25:49,845 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bb0fb38efb34301aea9c962cfab4972, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731878747535 2024-11-17T21:25:49,845 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e2794d01ac64d74ab4493f2fe3c0428, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731878748210 2024-11-17T21:25:49,845 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:49,849 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:49,849 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/B is initiating minor compaction (all files) 2024-11-17T21:25:49,849 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/B in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:49,850 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/6e0f8df1af7248a08f01c20e17dc6a23, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/5574695a425d47c7a8c527594164fa54, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/b9fc1757c9b74128ac8776c4d1c49d18] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=36.4 K 2024-11-17T21:25:49,850 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e0f8df1af7248a08f01c20e17dc6a23, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731878747330 2024-11-17T21:25:49,850 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 5574695a425d47c7a8c527594164fa54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731878747535 2024-11-17T21:25:49,851 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting b9fc1757c9b74128ac8776c4d1c49d18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731878748230 2024-11-17T21:25:49,877 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:49,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T21:25:49,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:49,879 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-17T21:25:49,898 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#B#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:49,898 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/fa751df7a04e48cebc795844d4892297 is 50, key is test_row_0/B:col10/1731878748888/Put/seqid=0 2024-11-17T21:25:49,900 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111784c355beb6b94590926df1e9f0ea55ff_c3ee12f1af122429c9b29c87eab606c8 store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:49,902 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111784c355beb6b94590926df1e9f0ea55ff_c3ee12f1af122429c9b29c87eab606c8, store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:49,902 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111784c355beb6b94590926df1e9f0ea55ff_c3ee12f1af122429c9b29c87eab606c8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:49,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fd710a43aec04f42a701fda6fda1aca7_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878748931/Put/seqid=0 2024-11-17T21:25:49,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742235_1411 (size=12949) 2024-11-17T21:25:49,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742237_1413 (size=12454) 2024-11-17T21:25:49,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:49,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742236_1412 (size=4469) 2024-11-17T21:25:49,990 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fd710a43aec04f42a701fda6fda1aca7_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fd710a43aec04f42a701fda6fda1aca7_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:49,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/e0e52011f3414406bf8ac27044aff2de, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:49,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/e0e52011f3414406bf8ac27044aff2de is 175, key is test_row_0/A:col10/1731878748931/Put/seqid=0 2024-11-17T21:25:50,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742238_1414 (size=31255) 2024-11-17T21:25:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T21:25:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:50,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:50,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878810167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:50,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:50,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878810169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:50,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878810170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:50,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878810172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:50,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:50,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878810281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:50,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:50,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878810291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:50,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878810291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:50,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:50,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878810291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:50,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:50,372 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/fa751df7a04e48cebc795844d4892297 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/fa751df7a04e48cebc795844d4892297 2024-11-17T21:25:50,376 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/B of c3ee12f1af122429c9b29c87eab606c8 into fa751df7a04e48cebc795844d4892297(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:50,376 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:50,376 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/B, priority=13, startTime=1731878749843; duration=0sec 2024-11-17T21:25:50,376 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:50,376 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:B 2024-11-17T21:25:50,376 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:50,377 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:50,377 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/C is initiating minor compaction (all files) 2024-11-17T21:25:50,377 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/C in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:50,377 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d73219b8b9a84d6d871e0b76cfad9f76, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/0e167a519dfc45aebc3d0378d00f4a66, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/2b7182d0d82e4a61b7ac9eae31f2e2d2] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=36.4 K 2024-11-17T21:25:50,378 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d73219b8b9a84d6d871e0b76cfad9f76, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731878747330 2024-11-17T21:25:50,378 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e167a519dfc45aebc3d0378d00f4a66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731878747535 2024-11-17T21:25:50,378 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b7182d0d82e4a61b7ac9eae31f2e2d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731878748230 2024-11-17T21:25:50,384 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#A#compaction#344 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:50,385 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/dced19df778049bcad346a7aca7a12eb is 175, key is test_row_0/A:col10/1731878748888/Put/seqid=0 2024-11-17T21:25:50,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742239_1415 (size=31903) 2024-11-17T21:25:50,389 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#C#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:50,389 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/d7d0e820247a402d8a2981101692d724 is 50, key is test_row_0/C:col10/1731878748888/Put/seqid=0 2024-11-17T21:25:50,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742240_1416 (size=12949) 2024-11-17T21:25:50,432 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/e0e52011f3414406bf8ac27044aff2de 2024-11-17T21:25:50,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/d25026154636468a92d7b967bf39e9b7 is 50, key is test_row_0/B:col10/1731878748931/Put/seqid=0 2024-11-17T21:25:50,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742241_1417 (size=12301) 2024-11-17T21:25:50,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878810491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:50,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878810511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:50,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878810511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878810512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:50,793 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/dced19df778049bcad346a7aca7a12eb as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/dced19df778049bcad346a7aca7a12eb 2024-11-17T21:25:50,806 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/A of c3ee12f1af122429c9b29c87eab606c8 into dced19df778049bcad346a7aca7a12eb(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:50,806 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:50,807 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/A, priority=13, startTime=1731878749842; duration=0sec 2024-11-17T21:25:50,807 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:50,807 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:A 2024-11-17T21:25:50,808 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/d7d0e820247a402d8a2981101692d724 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d7d0e820247a402d8a2981101692d724 2024-11-17T21:25:50,812 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/C of c3ee12f1af122429c9b29c87eab606c8 into d7d0e820247a402d8a2981101692d724(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:50,813 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:50,813 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/C, priority=13, startTime=1731878749843; duration=0sec 2024-11-17T21:25:50,813 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:50,813 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:C 2024-11-17T21:25:50,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878810801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:50,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:50,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878810831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878810831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:50,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878810831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:50,846 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/d25026154636468a92d7b967bf39e9b7 2024-11-17T21:25:50,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/6a71f48dcc6040f59541625139b66d10 is 50, key is test_row_0/C:col10/1731878748931/Put/seqid=0 2024-11-17T21:25:50,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742242_1418 (size=12301) 2024-11-17T21:25:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T21:25:51,273 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/6a71f48dcc6040f59541625139b66d10 2024-11-17T21:25:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/e0e52011f3414406bf8ac27044aff2de as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/e0e52011f3414406bf8ac27044aff2de 2024-11-17T21:25:51,295 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/e0e52011f3414406bf8ac27044aff2de, entries=150, sequenceid=309, filesize=30.5 K 2024-11-17T21:25:51,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/d25026154636468a92d7b967bf39e9b7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d25026154636468a92d7b967bf39e9b7 2024-11-17T21:25:51,300 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d25026154636468a92d7b967bf39e9b7, entries=150, sequenceid=309, filesize=12.0 K 2024-11-17T21:25:51,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/6a71f48dcc6040f59541625139b66d10 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/6a71f48dcc6040f59541625139b66d10 2024-11-17T21:25:51,305 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/6a71f48dcc6040f59541625139b66d10, entries=150, sequenceid=309, filesize=12.0 K 2024-11-17T21:25:51,306 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for c3ee12f1af122429c9b29c87eab606c8 in 1427ms, sequenceid=309, compaction requested=false 2024-11-17T21:25:51,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:51,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:51,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-17T21:25:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-17T21:25:51,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-17T21:25:51,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3550 sec 2024-11-17T21:25:51,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.3600 sec 2024-11-17T21:25:51,331 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:51,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-17T21:25:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:51,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111713cab619a31a443bbbed87ade4b67636_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878750168/Put/seqid=0 2024-11-17T21:25:51,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742243_1419 (size=12454) 2024-11-17T21:25:51,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878811405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:51,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:51,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878811405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:51,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878811406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:51,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878811420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:51,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878811531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878811532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:51,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878811532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:51,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:51,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878811541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:51,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:51,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878811751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:51,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878811751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:51,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878811753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:51,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878811753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:51,766 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:51,771 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111713cab619a31a443bbbed87ade4b67636_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111713cab619a31a443bbbed87ade4b67636_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:51,772 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/ce42a90dc8474bb5a5ee73b9614bc1d9, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:51,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/ce42a90dc8474bb5a5ee73b9614bc1d9 is 175, key is test_row_0/A:col10/1731878750168/Put/seqid=0 2024-11-17T21:25:51,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742244_1420 (size=31255) 2024-11-17T21:25:51,789 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/ce42a90dc8474bb5a5ee73b9614bc1d9 2024-11-17T21:25:51,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/00d085d0ae294f819e1188cff852ccf9 is 50, key is test_row_0/B:col10/1731878750168/Put/seqid=0 2024-11-17T21:25:51,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742245_1421 (size=12301) 2024-11-17T21:25:51,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/00d085d0ae294f819e1188cff852ccf9 2024-11-17T21:25:51,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/3234212f04774136b9cae13dfeb51430 is 50, key is test_row_0/C:col10/1731878750168/Put/seqid=0 2024-11-17T21:25:51,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742246_1422 (size=12301) 2024-11-17T21:25:51,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/3234212f04774136b9cae13dfeb51430 2024-11-17T21:25:51,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/ce42a90dc8474bb5a5ee73b9614bc1d9 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/ce42a90dc8474bb5a5ee73b9614bc1d9 2024-11-17T21:25:51,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/ce42a90dc8474bb5a5ee73b9614bc1d9, entries=150, sequenceid=329, filesize=30.5 K 2024-11-17T21:25:51,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/00d085d0ae294f819e1188cff852ccf9 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/00d085d0ae294f819e1188cff852ccf9 2024-11-17T21:25:51,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/00d085d0ae294f819e1188cff852ccf9, entries=150, sequenceid=329, filesize=12.0 K 2024-11-17T21:25:51,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/3234212f04774136b9cae13dfeb51430 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/3234212f04774136b9cae13dfeb51430 2024-11-17T21:25:51,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/3234212f04774136b9cae13dfeb51430, entries=150, sequenceid=329, filesize=12.0 K 2024-11-17T21:25:51,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for c3ee12f1af122429c9b29c87eab606c8 in 554ms, sequenceid=329, compaction requested=true 2024-11-17T21:25:51,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:51,885 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:51,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:25:51,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:51,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:51,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:51,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:25:51,886 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:51,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:51,887 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:51,887 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/A is initiating minor compaction (all files) 2024-11-17T21:25:51,887 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/A in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:51,887 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:51,887 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/B is initiating minor compaction (all files) 2024-11-17T21:25:51,887 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/dced19df778049bcad346a7aca7a12eb, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/e0e52011f3414406bf8ac27044aff2de, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/ce42a90dc8474bb5a5ee73b9614bc1d9] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=92.2 K 2024-11-17T21:25:51,887 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/B in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:51,887 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:51,887 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/dced19df778049bcad346a7aca7a12eb, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/e0e52011f3414406bf8ac27044aff2de, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/ce42a90dc8474bb5a5ee73b9614bc1d9] 2024-11-17T21:25:51,887 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/fa751df7a04e48cebc795844d4892297, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d25026154636468a92d7b967bf39e9b7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/00d085d0ae294f819e1188cff852ccf9] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=36.7 K 2024-11-17T21:25:51,888 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting fa751df7a04e48cebc795844d4892297, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731878748230 2024-11-17T21:25:51,888 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting dced19df778049bcad346a7aca7a12eb, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731878748230 2024-11-17T21:25:51,889 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0e52011f3414406bf8ac27044aff2de, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731878748919 2024-11-17T21:25:51,889 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d25026154636468a92d7b967bf39e9b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731878748919 2024-11-17T21:25:51,890 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce42a90dc8474bb5a5ee73b9614bc1d9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731878750168 2024-11-17T21:25:51,890 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 00d085d0ae294f819e1188cff852ccf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731878750168 2024-11-17T21:25:51,906 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#B#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:51,907 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/d13c4f497e7b4507b5a249f3b6823104 is 50, key is test_row_0/B:col10/1731878750168/Put/seqid=0 2024-11-17T21:25:51,913 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:51,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742247_1423 (size=13051) 2024-11-17T21:25:51,961 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117900509011f864188a3ebf536082a7cc3_c3ee12f1af122429c9b29c87eab606c8 store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:51,963 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117900509011f864188a3ebf536082a7cc3_c3ee12f1af122429c9b29c87eab606c8, store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:51,963 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117900509011f864188a3ebf536082a7cc3_c3ee12f1af122429c9b29c87eab606c8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:52,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742248_1424 (size=4469) 2024-11-17T21:25:52,077 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:52,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-17T21:25:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:52,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cf91ca16ac0741bea21a332e0e1933a5_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878751418/Put/seqid=0 2024-11-17T21:25:52,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742249_1425 (size=14994) 2024-11-17T21:25:52,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878812148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:52,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878812150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:52,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878812165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:52,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878812166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:52,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:52,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878812272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:52,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878812272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:52,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878812282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:52,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878812291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:52,364 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/d13c4f497e7b4507b5a249f3b6823104 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d13c4f497e7b4507b5a249f3b6823104 2024-11-17T21:25:52,381 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/B of c3ee12f1af122429c9b29c87eab606c8 into d13c4f497e7b4507b5a249f3b6823104(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:52,381 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:52,381 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/B, priority=13, startTime=1731878751886; duration=0sec 2024-11-17T21:25:52,381 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:52,381 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:B 2024-11-17T21:25:52,381 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:25:52,382 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:25:52,382 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/C is initiating minor compaction (all files) 2024-11-17T21:25:52,382 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/C in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:52,383 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d7d0e820247a402d8a2981101692d724, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/6a71f48dcc6040f59541625139b66d10, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/3234212f04774136b9cae13dfeb51430] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=36.7 K 2024-11-17T21:25:52,383 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d7d0e820247a402d8a2981101692d724, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731878748230 2024-11-17T21:25:52,383 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a71f48dcc6040f59541625139b66d10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731878748919 2024-11-17T21:25:52,383 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 3234212f04774136b9cae13dfeb51430, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731878750168 2024-11-17T21:25:52,391 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#C#compaction#356 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:52,392 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/ca0605c97ca04239a7247f7ad9566309 is 50, key is test_row_0/C:col10/1731878750168/Put/seqid=0 2024-11-17T21:25:52,416 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#A#compaction#354 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:52,416 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/2aae0b115329453ba0735e3bba690173 is 175, key is test_row_0/A:col10/1731878750168/Put/seqid=0 2024-11-17T21:25:52,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742250_1426 (size=13051) 2024-11-17T21:25:52,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742251_1427 (size=32005) 2024-11-17T21:25:52,442 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/2aae0b115329453ba0735e3bba690173 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2aae0b115329453ba0735e3bba690173 2024-11-17T21:25:52,446 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/A of c3ee12f1af122429c9b29c87eab606c8 into 2aae0b115329453ba0735e3bba690173(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:52,447 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:52,447 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/A, priority=13, startTime=1731878751885; duration=0sec 2024-11-17T21:25:52,447 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:52,447 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:A 2024-11-17T21:25:52,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878812481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:52,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878812481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:52,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:52,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:52,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878812492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:52,509 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:52,515 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cf91ca16ac0741bea21a332e0e1933a5_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cf91ca16ac0741bea21a332e0e1933a5_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:52,516 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/2f1ed91c12b84ec5ad9a6995caa5908c, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:52,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/2f1ed91c12b84ec5ad9a6995caa5908c is 175, key is test_row_0/A:col10/1731878751418/Put/seqid=0 2024-11-17T21:25:52,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878812511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:52,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742252_1428 (size=39949) 2024-11-17T21:25:52,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878812801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878812801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:52,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878812811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:52,830 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/ca0605c97ca04239a7247f7ad9566309 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/ca0605c97ca04239a7247f7ad9566309 2024-11-17T21:25:52,838 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/C of c3ee12f1af122429c9b29c87eab606c8 into ca0605c97ca04239a7247f7ad9566309(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:52,838 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:52,839 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/C, priority=13, startTime=1731878751886; duration=0sec 2024-11-17T21:25:52,839 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:52,839 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:C 2024-11-17T21:25:52,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878812832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:52,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:52,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:52,950 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/2f1ed91c12b84ec5ad9a6995caa5908c 2024-11-17T21:25:52,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/3ae0fe8849e1436d8a6e8b92db91a72b is 50, key is test_row_0/B:col10/1731878751418/Put/seqid=0 2024-11-17T21:25:52,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742253_1429 (size=12301) 2024-11-17T21:25:52,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/3ae0fe8849e1436d8a6e8b92db91a72b 2024-11-17T21:25:52,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/2abab10a3ff6461e9d5d1b028f314edd is 50, key is test_row_0/C:col10/1731878751418/Put/seqid=0 2024-11-17T21:25:53,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742254_1430 (size=12301) 2024-11-17T21:25:53,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/2abab10a3ff6461e9d5d1b028f314edd 2024-11-17T21:25:53,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/2f1ed91c12b84ec5ad9a6995caa5908c as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2f1ed91c12b84ec5ad9a6995caa5908c 2024-11-17T21:25:53,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2f1ed91c12b84ec5ad9a6995caa5908c, entries=200, sequenceid=349, filesize=39.0 K 2024-11-17T21:25:53,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/3ae0fe8849e1436d8a6e8b92db91a72b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/3ae0fe8849e1436d8a6e8b92db91a72b 2024-11-17T21:25:53,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/3ae0fe8849e1436d8a6e8b92db91a72b, entries=150, sequenceid=349, filesize=12.0 K 2024-11-17T21:25:53,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/2abab10a3ff6461e9d5d1b028f314edd as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/2abab10a3ff6461e9d5d1b028f314edd 2024-11-17T21:25:53,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/2abab10a3ff6461e9d5d1b028f314edd, entries=150, sequenceid=349, filesize=12.0 K 2024-11-17T21:25:53,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for c3ee12f1af122429c9b29c87eab606c8 in 998ms, sequenceid=349, compaction requested=false 2024-11-17T21:25:53,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T21:25:53,131 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:53,132 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:25:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-17T21:25:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T21:25:53,133 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:25:53,134 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:25:53,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:25:53,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T21:25:53,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T21:25:53,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:53,288 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T21:25:53,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178f623284afd844f2b88cfd8917a268df_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878752163/Put/seqid=0 2024-11-17T21:25:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:53,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. as already flushing 2024-11-17T21:25:53,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742255_1431 (size=12454) 2024-11-17T21:25:53,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878813390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:53,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878813391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:53,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:53,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878813402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:53,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:53,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878813403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:53,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1731878813420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:53,432 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_0' in c3ee12f1af122429c9b29c87eab606c8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8317 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:25:53,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:53,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T21:25:53,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878813511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:53,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878813511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:53,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:53,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878813521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:53,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878813521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:53,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731878813721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:53,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55694 deadline: 1731878813721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:53,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:53,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:53,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:53,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:53,736 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178f623284afd844f2b88cfd8917a268df_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178f623284afd844f2b88cfd8917a268df_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55634 deadline: 1731878813731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:25:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55622 deadline: 1731878813731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 2024-11-17T21:25:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:25:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:25:53,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/038c91d07b184096b7f9f899d6954c6b, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:53,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/038c91d07b184096b7f9f899d6954c6b is 175, key is test_row_0/A:col10/1731878752163/Put/seqid=0 2024-11-17T21:25:53,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742256_1432 (size=31255) 2024-11-17T21:25:53,747 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/038c91d07b184096b7f9f899d6954c6b 2024-11-17T21:25:53,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/425b0739a4e6418ca2215cd215e306b4 is 50, key is test_row_0/B:col10/1731878752163/Put/seqid=0 2024-11-17T21:25:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T21:25:53,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742257_1433 (size=12301) 2024-11-17T21:25:53,774 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/425b0739a4e6418ca2215cd215e306b4 2024-11-17T21:25:53,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/aaf69db593644847bc697f378274a7e7 is 50, key is test_row_0/C:col10/1731878752163/Put/seqid=0 2024-11-17T21:25:53,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742258_1434 (size=12301) 2024-11-17T21:25:53,799 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/aaf69db593644847bc697f378274a7e7 2024-11-17T21:25:53,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/038c91d07b184096b7f9f899d6954c6b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/038c91d07b184096b7f9f899d6954c6b 2024-11-17T21:25:53,818 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/038c91d07b184096b7f9f899d6954c6b, entries=150, sequenceid=368, filesize=30.5 K 2024-11-17T21:25:53,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/425b0739a4e6418ca2215cd215e306b4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/425b0739a4e6418ca2215cd215e306b4 2024-11-17T21:25:53,823 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/425b0739a4e6418ca2215cd215e306b4, entries=150, sequenceid=368, filesize=12.0 K 2024-11-17T21:25:53,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/aaf69db593644847bc697f378274a7e7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/aaf69db593644847bc697f378274a7e7 2024-11-17T21:25:53,828 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/aaf69db593644847bc697f378274a7e7, entries=150, sequenceid=368, filesize=12.0 K 2024-11-17T21:25:53,829 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for c3ee12f1af122429c9b29c87eab606c8 in 541ms, sequenceid=368, compaction requested=true 2024-11-17T21:25:53,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:53,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:53,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-17T21:25:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-17T21:25:53,832 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-17T21:25:53,832 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 696 msec 2024-11-17T21:25:53,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 700 msec 2024-11-17T21:25:53,903 INFO [Thread-1539 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1539. 2024-11-17T21:25:53,903 DEBUG [Thread-1539 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:53,903 DEBUG [Thread-1539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:53,904 DEBUG [Thread-1539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:53,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:53,913 INFO [Thread-1537 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1537. 2024-11-17T21:25:53,913 DEBUG [Thread-1537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:53,913 DEBUG [Thread-1537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:53,913 DEBUG [Thread-1537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:53,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:53,914 INFO [Thread-1543 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1543. 2024-11-17T21:25:53,914 DEBUG [Thread-1543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:53,914 DEBUG [Thread-1543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:53,914 DEBUG [Thread-1543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:53,914 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:53,917 INFO [Thread-1535 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1535. 2024-11-17T21:25:53,917 DEBUG [Thread-1535 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:53,917 DEBUG [Thread-1535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:53,917 DEBUG [Thread-1535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:53,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:53,918 INFO [Thread-1541 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1541. 2024-11-17T21:25:53,918 DEBUG [Thread-1541 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:53,918 DEBUG [Thread-1541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:53,919 DEBUG [Thread-1541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:53,919 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:54,033 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:25:54,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T21:25:54,033 INFO [Thread-1532 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1532. 2024-11-17T21:25:54,033 INFO [Thread-1524 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1524. 2024-11-17T21:25:54,033 DEBUG [Thread-1532 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:54,033 DEBUG [Thread-1524 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:54,033 DEBUG [Thread-1532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:54,033 DEBUG [Thread-1524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:54,033 DEBUG [Thread-1532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:54,033 DEBUG [Thread-1524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:54,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:54,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:54,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ba25fb503c1d4d2cabe5ef5edc950e28_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878753400/Put/seqid=0 2024-11-17T21:25:54,043 INFO [Thread-1528 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1528. 2024-11-17T21:25:54,043 DEBUG [Thread-1528 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:54,043 DEBUG [Thread-1528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:54,043 DEBUG [Thread-1528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:54,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:54,044 INFO [Thread-1530 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1530. 2024-11-17T21:25:54,044 DEBUG [Thread-1530 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:25:54,044 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:54,044 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:25:54,044 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:25:54,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742259_1435 (size=12454) 2024-11-17T21:25:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T21:25:54,272 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:25:54,447 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:25:54,450 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ba25fb503c1d4d2cabe5ef5edc950e28_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ba25fb503c1d4d2cabe5ef5edc950e28_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:25:54,451 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/79cd8919d5fb462cac266b231aab2624, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:54,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/79cd8919d5fb462cac266b231aab2624 is 175, key is test_row_0/A:col10/1731878753400/Put/seqid=0 2024-11-17T21:25:54,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742260_1436 (size=31255) 2024-11-17T21:25:54,855 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=390, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/79cd8919d5fb462cac266b231aab2624 2024-11-17T21:25:54,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/c1b7d29a26be4fa393fe93caab2d82b6 is 50, key is test_row_0/B:col10/1731878753400/Put/seqid=0 2024-11-17T21:25:54,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742261_1437 (size=12301) 2024-11-17T21:25:55,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/c1b7d29a26be4fa393fe93caab2d82b6 2024-11-17T21:25:55,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/aa1d9e27905c4aafaab6bc83e8705665 is 50, key is test_row_0/C:col10/1731878753400/Put/seqid=0 2024-11-17T21:25:55,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742262_1438 (size=12301) 2024-11-17T21:25:55,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/aa1d9e27905c4aafaab6bc83e8705665 2024-11-17T21:25:55,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/79cd8919d5fb462cac266b231aab2624 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/79cd8919d5fb462cac266b231aab2624 2024-11-17T21:25:55,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/79cd8919d5fb462cac266b231aab2624, entries=150, sequenceid=390, filesize=30.5 K 2024-11-17T21:25:55,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/c1b7d29a26be4fa393fe93caab2d82b6 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/c1b7d29a26be4fa393fe93caab2d82b6 2024-11-17T21:25:55,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/c1b7d29a26be4fa393fe93caab2d82b6, entries=150, sequenceid=390, filesize=12.0 K 2024-11-17T21:25:55,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/aa1d9e27905c4aafaab6bc83e8705665 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/aa1d9e27905c4aafaab6bc83e8705665 2024-11-17T21:25:55,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/aa1d9e27905c4aafaab6bc83e8705665, entries=150, sequenceid=390, filesize=12.0 K 2024-11-17T21:25:55,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=13.42 KB/13740 for c3ee12f1af122429c9b29c87eab606c8 in 1674ms, sequenceid=390, compaction requested=true 2024-11-17T21:25:55,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:55,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:25:55,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:55,708 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:55,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:25:55,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:55,708 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:55,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3ee12f1af122429c9b29c87eab606c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:25:55,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:55,708 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:55,708 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:55,708 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/B is initiating minor compaction (all files) 2024-11-17T21:25:55,708 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/A is initiating minor compaction (all files) 2024-11-17T21:25:55,708 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/B in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:55,708 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/A in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:55,709 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d13c4f497e7b4507b5a249f3b6823104, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/3ae0fe8849e1436d8a6e8b92db91a72b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/425b0739a4e6418ca2215cd215e306b4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/c1b7d29a26be4fa393fe93caab2d82b6] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=48.8 K 2024-11-17T21:25:55,709 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2aae0b115329453ba0735e3bba690173, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2f1ed91c12b84ec5ad9a6995caa5908c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/038c91d07b184096b7f9f899d6954c6b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/79cd8919d5fb462cac266b231aab2624] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=131.3 K 2024-11-17T21:25:55,709 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:55,709 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2aae0b115329453ba0735e3bba690173, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2f1ed91c12b84ec5ad9a6995caa5908c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/038c91d07b184096b7f9f899d6954c6b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/79cd8919d5fb462cac266b231aab2624] 2024-11-17T21:25:55,709 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d13c4f497e7b4507b5a249f3b6823104, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731878750168 2024-11-17T21:25:55,709 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2aae0b115329453ba0735e3bba690173, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731878750168 2024-11-17T21:25:55,709 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ae0fe8849e1436d8a6e8b92db91a72b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731878751403 2024-11-17T21:25:55,709 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f1ed91c12b84ec5ad9a6995caa5908c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731878751402 2024-11-17T21:25:55,709 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 425b0739a4e6418ca2215cd215e306b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731878752148 2024-11-17T21:25:55,709 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 038c91d07b184096b7f9f899d6954c6b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731878752148 2024-11-17T21:25:55,709 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting c1b7d29a26be4fa393fe93caab2d82b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731878753400 2024-11-17T21:25:55,709 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79cd8919d5fb462cac266b231aab2624, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731878753400 2024-11-17T21:25:55,713 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:55,714 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411176b1a5964aa374a029261a2db69be18bf_c3ee12f1af122429c9b29c87eab606c8 store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:55,715 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#B#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:55,715 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/aedba1a6c6f148fd96a65651885dc98e is 50, key is test_row_0/B:col10/1731878753400/Put/seqid=0 2024-11-17T21:25:55,719 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411176b1a5964aa374a029261a2db69be18bf_c3ee12f1af122429c9b29c87eab606c8, store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:55,720 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176b1a5964aa374a029261a2db69be18bf_c3ee12f1af122429c9b29c87eab606c8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:25:55,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742263_1439 (size=13187) 2024-11-17T21:25:55,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742264_1440 (size=4469) 2024-11-17T21:25:56,134 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/aedba1a6c6f148fd96a65651885dc98e as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/aedba1a6c6f148fd96a65651885dc98e 2024-11-17T21:25:56,137 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#A#compaction#366 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:56,138 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/7f446cfb30b64e28ac6175e2d5600687 is 175, key is test_row_0/A:col10/1731878753400/Put/seqid=0 2024-11-17T21:25:56,139 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/B of c3ee12f1af122429c9b29c87eab606c8 into aedba1a6c6f148fd96a65651885dc98e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:56,139 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:56,139 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/B, priority=12, startTime=1731878755708; duration=0sec 2024-11-17T21:25:56,139 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:25:56,139 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:B 2024-11-17T21:25:56,139 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:25:56,141 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:25:56,141 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): c3ee12f1af122429c9b29c87eab606c8/C is initiating minor compaction (all files) 2024-11-17T21:25:56,141 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3ee12f1af122429c9b29c87eab606c8/C in TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:25:56,141 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/ca0605c97ca04239a7247f7ad9566309, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/2abab10a3ff6461e9d5d1b028f314edd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/aaf69db593644847bc697f378274a7e7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/aa1d9e27905c4aafaab6bc83e8705665] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp, totalSize=48.8 K 2024-11-17T21:25:56,141 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting ca0605c97ca04239a7247f7ad9566309, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731878750168 2024-11-17T21:25:56,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742265_1441 (size=32141) 2024-11-17T21:25:56,142 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 2abab10a3ff6461e9d5d1b028f314edd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731878751403 2024-11-17T21:25:56,142 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting aaf69db593644847bc697f378274a7e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731878752148 2024-11-17T21:25:56,143 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting aa1d9e27905c4aafaab6bc83e8705665, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731878753400 2024-11-17T21:25:56,150 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3ee12f1af122429c9b29c87eab606c8#C#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:25:56,151 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/75f5d81fd5c24828927192b7fd8f46c1 is 50, key is test_row_0/C:col10/1731878753400/Put/seqid=0 2024-11-17T21:25:56,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742266_1442 (size=13187) 2024-11-17T21:25:56,551 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/7f446cfb30b64e28ac6175e2d5600687 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/7f446cfb30b64e28ac6175e2d5600687 2024-11-17T21:25:56,560 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/A of c3ee12f1af122429c9b29c87eab606c8 into 7f446cfb30b64e28ac6175e2d5600687(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:56,560 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:56,560 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/A, priority=12, startTime=1731878755707; duration=0sec 2024-11-17T21:25:56,560 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:56,560 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:A 2024-11-17T21:25:56,562 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/75f5d81fd5c24828927192b7fd8f46c1 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/75f5d81fd5c24828927192b7fd8f46c1 2024-11-17T21:25:56,566 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c3ee12f1af122429c9b29c87eab606c8/C of c3ee12f1af122429c9b29c87eab606c8 into 75f5d81fd5c24828927192b7fd8f46c1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:25:56,566 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3ee12f1af122429c9b29c87eab606c8: 2024-11-17T21:25:56,566 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., storeName=c3ee12f1af122429c9b29c87eab606c8/C, priority=12, startTime=1731878755708; duration=0sec 2024-11-17T21:25:56,567 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:25:56,567 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3ee12f1af122429c9b29c87eab606c8:C 2024-11-17T21:26:01,090 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T21:26:03,513 INFO [Thread-1526 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1526. 2024-11-17T21:26:03,514 DEBUG [Thread-1526 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:03,514 DEBUG [Thread-1526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:03,514 DEBUG [Thread-1526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:03,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:03,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T21:26:03,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-17T21:26:03,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-17T21:26:03,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-17T21:26:03,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-17T21:26:03,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-17T21:26:03,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T21:26:03,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T21:26:03,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1096 2024-11-17T21:26:03,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3288 rows 2024-11-17T21:26:03,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1086 2024-11-17T21:26:03,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3256 rows 2024-11-17T21:26:03,515 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1096 2024-11-17T21:26:03,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3288 rows 2024-11-17T21:26:03,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1099 2024-11-17T21:26:03,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3297 rows 2024-11-17T21:26:03,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1088 2024-11-17T21:26:03,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3259 rows 2024-11-17T21:26:03,516 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T21:26:03,516 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T21:26:03,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:03,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:03,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:03,519 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T21:26:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T21:26:03,523 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878763523"}]},"ts":"1731878763523"} 2024-11-17T21:26:03,524 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T21:26:03,524 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T21:26:03,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T21:26:03,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c3ee12f1af122429c9b29c87eab606c8, UNASSIGN}] 2024-11-17T21:26:03,526 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c3ee12f1af122429c9b29c87eab606c8, UNASSIGN 2024-11-17T21:26:03,527 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=c3ee12f1af122429c9b29c87eab606c8, regionState=CLOSING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:26:03,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c3ee12f1af122429c9b29c87eab606c8, UNASSIGN because future has completed 2024-11-17T21:26:03,528 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T21:26:03,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:26:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T21:26:03,681 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:03,681 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T21:26:03,681 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing c3ee12f1af122429c9b29c87eab606c8, disabling compactions & flushes 2024-11-17T21:26:03,681 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:26:03,681 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:26:03,681 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. after waiting 0 ms 2024-11-17T21:26:03,681 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:26:03,682 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing c3ee12f1af122429c9b29c87eab606c8 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-17T21:26:03,693 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117da69c9d94b494549b109c7263d488ca1_c3ee12f1af122429c9b29c87eab606c8 is 50, key is test_row_0/A:col10/1731878763463/Put/seqid=0 2024-11-17T21:26:03,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742267_1443 (size=9914) 2024-11-17T21:26:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T21:26:04,098 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:04,108 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117da69c9d94b494549b109c7263d488ca1_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117da69c9d94b494549b109c7263d488ca1_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:04,109 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/b8c5061309704e9c8e517f3a3d4f86ce, store: [table=TestAcidGuarantees family=A region=c3ee12f1af122429c9b29c87eab606c8] 2024-11-17T21:26:04,110 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/b8c5061309704e9c8e517f3a3d4f86ce is 175, key is test_row_0/A:col10/1731878763463/Put/seqid=0 2024-11-17T21:26:04,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742268_1444 (size=22561) 2024-11-17T21:26:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T21:26:04,516 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/b8c5061309704e9c8e517f3a3d4f86ce 2024-11-17T21:26:04,530 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/538e35551f124267ad734dd572aa02f0 is 50, key is test_row_0/B:col10/1731878763463/Put/seqid=0 2024-11-17T21:26:04,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742269_1445 (size=9857) 2024-11-17T21:26:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T21:26:04,935 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/538e35551f124267ad734dd572aa02f0 2024-11-17T21:26:04,948 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/ed43fc5c33cf4459a615e00046077465 is 50, key is test_row_0/C:col10/1731878763463/Put/seqid=0 2024-11-17T21:26:04,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742270_1446 (size=9857) 2024-11-17T21:26:05,353 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/ed43fc5c33cf4459a615e00046077465 2024-11-17T21:26:05,364 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/A/b8c5061309704e9c8e517f3a3d4f86ce as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/b8c5061309704e9c8e517f3a3d4f86ce 2024-11-17T21:26:05,368 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/b8c5061309704e9c8e517f3a3d4f86ce, entries=100, sequenceid=399, filesize=22.0 K 2024-11-17T21:26:05,369 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/B/538e35551f124267ad734dd572aa02f0 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/538e35551f124267ad734dd572aa02f0 2024-11-17T21:26:05,373 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/538e35551f124267ad734dd572aa02f0, entries=100, sequenceid=399, filesize=9.6 K 2024-11-17T21:26:05,374 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/.tmp/C/ed43fc5c33cf4459a615e00046077465 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/ed43fc5c33cf4459a615e00046077465 2024-11-17T21:26:05,378 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/ed43fc5c33cf4459a615e00046077465, entries=100, sequenceid=399, filesize=9.6 K 2024-11-17T21:26:05,379 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for c3ee12f1af122429c9b29c87eab606c8 in 1697ms, sequenceid=399, compaction requested=false 2024-11-17T21:26:05,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c47bb3ebde3245858a06b2d1ae3295ff, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/5960681efc3d4beebb062d81b2b0fa4c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/ecf9ea3a103f427fa145753664c0c593, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/a59390235d634245b2e5dba4abedd33d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/f609e31113424eafaebf8914ffd9d9df, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c2187f376ddd4904ae35a41119d49522, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/229a739341324de8a362564c346a0460, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/f1bca9b5083f4fe9b4786b0fd885ae25, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/0af91247a6ee432a989cae4b2fed8154, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/b4e4b4a6458b4eec955394aad1fb6844, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/06a968e631374cb8991575ed8dbb27c9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2a0ecdeada3e4c11ae266f15d9ab0858, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/14acceec788a471ea255ae9d8729cc7a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c8d38c893a0e45429e835e367ba0fa4c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/d20da1981dad4abeb6ce42fd382a0f2b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/9614ed06981c4bc39e9be92af5efad51, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/0554d45f61074c7dbc97e5248fa30d7a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/1bb0fb38efb34301aea9c962cfab4972, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/1e2794d01ac64d74ab4493f2fe3c0428, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/dced19df778049bcad346a7aca7a12eb, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/e0e52011f3414406bf8ac27044aff2de, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2aae0b115329453ba0735e3bba690173, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/ce42a90dc8474bb5a5ee73b9614bc1d9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2f1ed91c12b84ec5ad9a6995caa5908c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/038c91d07b184096b7f9f899d6954c6b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/79cd8919d5fb462cac266b231aab2624] to archive 2024-11-17T21:26:05,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:26:05,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c47bb3ebde3245858a06b2d1ae3295ff to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c47bb3ebde3245858a06b2d1ae3295ff 2024-11-17T21:26:05,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/5960681efc3d4beebb062d81b2b0fa4c to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/5960681efc3d4beebb062d81b2b0fa4c 2024-11-17T21:26:05,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/ecf9ea3a103f427fa145753664c0c593 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/ecf9ea3a103f427fa145753664c0c593 2024-11-17T21:26:05,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/a59390235d634245b2e5dba4abedd33d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/a59390235d634245b2e5dba4abedd33d 2024-11-17T21:26:05,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/f609e31113424eafaebf8914ffd9d9df to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/f609e31113424eafaebf8914ffd9d9df 2024-11-17T21:26:05,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c2187f376ddd4904ae35a41119d49522 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c2187f376ddd4904ae35a41119d49522 2024-11-17T21:26:05,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/229a739341324de8a362564c346a0460 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/229a739341324de8a362564c346a0460 2024-11-17T21:26:05,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/f1bca9b5083f4fe9b4786b0fd885ae25 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/f1bca9b5083f4fe9b4786b0fd885ae25 2024-11-17T21:26:05,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/0af91247a6ee432a989cae4b2fed8154 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/0af91247a6ee432a989cae4b2fed8154 2024-11-17T21:26:05,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/b4e4b4a6458b4eec955394aad1fb6844 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/b4e4b4a6458b4eec955394aad1fb6844 2024-11-17T21:26:05,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/06a968e631374cb8991575ed8dbb27c9 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/06a968e631374cb8991575ed8dbb27c9 2024-11-17T21:26:05,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2a0ecdeada3e4c11ae266f15d9ab0858 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2a0ecdeada3e4c11ae266f15d9ab0858 2024-11-17T21:26:05,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/14acceec788a471ea255ae9d8729cc7a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/14acceec788a471ea255ae9d8729cc7a 2024-11-17T21:26:05,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c8d38c893a0e45429e835e367ba0fa4c to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/c8d38c893a0e45429e835e367ba0fa4c 2024-11-17T21:26:05,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/d20da1981dad4abeb6ce42fd382a0f2b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/d20da1981dad4abeb6ce42fd382a0f2b 2024-11-17T21:26:05,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/9614ed06981c4bc39e9be92af5efad51 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/9614ed06981c4bc39e9be92af5efad51 2024-11-17T21:26:05,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/0554d45f61074c7dbc97e5248fa30d7a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/0554d45f61074c7dbc97e5248fa30d7a 2024-11-17T21:26:05,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/1bb0fb38efb34301aea9c962cfab4972 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/1bb0fb38efb34301aea9c962cfab4972 2024-11-17T21:26:05,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/1e2794d01ac64d74ab4493f2fe3c0428 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/1e2794d01ac64d74ab4493f2fe3c0428 2024-11-17T21:26:05,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/dced19df778049bcad346a7aca7a12eb to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/dced19df778049bcad346a7aca7a12eb 2024-11-17T21:26:05,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/e0e52011f3414406bf8ac27044aff2de to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/e0e52011f3414406bf8ac27044aff2de 2024-11-17T21:26:05,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2aae0b115329453ba0735e3bba690173 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2aae0b115329453ba0735e3bba690173 2024-11-17T21:26:05,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/ce42a90dc8474bb5a5ee73b9614bc1d9 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/ce42a90dc8474bb5a5ee73b9614bc1d9 2024-11-17T21:26:05,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2f1ed91c12b84ec5ad9a6995caa5908c to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/2f1ed91c12b84ec5ad9a6995caa5908c 2024-11-17T21:26:05,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/038c91d07b184096b7f9f899d6954c6b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/038c91d07b184096b7f9f899d6954c6b 2024-11-17T21:26:05,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/79cd8919d5fb462cac266b231aab2624 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/79cd8919d5fb462cac266b231aab2624 2024-11-17T21:26:05,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/8c9a24af8a3f4a1586c6b6320236bb15, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/a51894c33d574a1c8a055ff434e0431c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ebe6e4b9fee945e888e7b832c74cefc1, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/2053d16cfb6349a7bdd6855764da2999, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/4ffc3dfe4b3c49b7a5676b13f13c588b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/8f95d7f5bc4245ebbe02faec13314310, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/918ff5d630a442d5b78af4dbd29bad1e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ef8e07b98fe14e24b55e2efecbfae23f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d5a6095e5737413eba3881aa74bd6971, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/342099b99b7243a1ba188e82cc3d8cb3, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/7f2647ad91794996b8721fdba233c9ec, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/0fca2501198a4393bf6db71007edd43a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/77c9a38585fe4107af205146f8583589, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ad393b446b334f9d894cf4e272131024, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d923995d2d4a4a4f9895f6f54c2db9e5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/6e0f8df1af7248a08f01c20e17dc6a23, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/23431f91ce1341679c0659a32e015a52, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/5574695a425d47c7a8c527594164fa54, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/fa751df7a04e48cebc795844d4892297, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/b9fc1757c9b74128ac8776c4d1c49d18, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d25026154636468a92d7b967bf39e9b7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d13c4f497e7b4507b5a249f3b6823104, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/00d085d0ae294f819e1188cff852ccf9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/3ae0fe8849e1436d8a6e8b92db91a72b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/425b0739a4e6418ca2215cd215e306b4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/c1b7d29a26be4fa393fe93caab2d82b6] to archive 2024-11-17T21:26:05,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:26:05,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/8c9a24af8a3f4a1586c6b6320236bb15 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/8c9a24af8a3f4a1586c6b6320236bb15 2024-11-17T21:26:05,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/a51894c33d574a1c8a055ff434e0431c to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/a51894c33d574a1c8a055ff434e0431c 2024-11-17T21:26:05,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ebe6e4b9fee945e888e7b832c74cefc1 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ebe6e4b9fee945e888e7b832c74cefc1 2024-11-17T21:26:05,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/2053d16cfb6349a7bdd6855764da2999 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/2053d16cfb6349a7bdd6855764da2999 2024-11-17T21:26:05,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/4ffc3dfe4b3c49b7a5676b13f13c588b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/4ffc3dfe4b3c49b7a5676b13f13c588b 2024-11-17T21:26:05,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/8f95d7f5bc4245ebbe02faec13314310 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/8f95d7f5bc4245ebbe02faec13314310 2024-11-17T21:26:05,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/918ff5d630a442d5b78af4dbd29bad1e to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/918ff5d630a442d5b78af4dbd29bad1e 2024-11-17T21:26:05,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ef8e07b98fe14e24b55e2efecbfae23f to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ef8e07b98fe14e24b55e2efecbfae23f 2024-11-17T21:26:05,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d5a6095e5737413eba3881aa74bd6971 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d5a6095e5737413eba3881aa74bd6971 2024-11-17T21:26:05,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/342099b99b7243a1ba188e82cc3d8cb3 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/342099b99b7243a1ba188e82cc3d8cb3 2024-11-17T21:26:05,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/7f2647ad91794996b8721fdba233c9ec to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/7f2647ad91794996b8721fdba233c9ec 2024-11-17T21:26:05,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/0fca2501198a4393bf6db71007edd43a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/0fca2501198a4393bf6db71007edd43a 2024-11-17T21:26:05,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/77c9a38585fe4107af205146f8583589 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/77c9a38585fe4107af205146f8583589 2024-11-17T21:26:05,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ad393b446b334f9d894cf4e272131024 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/ad393b446b334f9d894cf4e272131024 2024-11-17T21:26:05,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d923995d2d4a4a4f9895f6f54c2db9e5 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d923995d2d4a4a4f9895f6f54c2db9e5 2024-11-17T21:26:05,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/6e0f8df1af7248a08f01c20e17dc6a23 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/6e0f8df1af7248a08f01c20e17dc6a23 2024-11-17T21:26:05,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/23431f91ce1341679c0659a32e015a52 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/23431f91ce1341679c0659a32e015a52 2024-11-17T21:26:05,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/5574695a425d47c7a8c527594164fa54 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/5574695a425d47c7a8c527594164fa54 2024-11-17T21:26:05,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/fa751df7a04e48cebc795844d4892297 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/fa751df7a04e48cebc795844d4892297 2024-11-17T21:26:05,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/b9fc1757c9b74128ac8776c4d1c49d18 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/b9fc1757c9b74128ac8776c4d1c49d18 2024-11-17T21:26:05,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d25026154636468a92d7b967bf39e9b7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d25026154636468a92d7b967bf39e9b7 2024-11-17T21:26:05,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d13c4f497e7b4507b5a249f3b6823104 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/d13c4f497e7b4507b5a249f3b6823104 2024-11-17T21:26:05,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/00d085d0ae294f819e1188cff852ccf9 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/00d085d0ae294f819e1188cff852ccf9 2024-11-17T21:26:05,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/3ae0fe8849e1436d8a6e8b92db91a72b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/3ae0fe8849e1436d8a6e8b92db91a72b 2024-11-17T21:26:05,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/425b0739a4e6418ca2215cd215e306b4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/425b0739a4e6418ca2215cd215e306b4 2024-11-17T21:26:05,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/c1b7d29a26be4fa393fe93caab2d82b6 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/c1b7d29a26be4fa393fe93caab2d82b6 2024-11-17T21:26:05,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/c15be5fb8657466abcd37ee0701964a7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/4e07432d038d43b6a59d5f912eea0fcd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/14f7bf4893e24f2490c0eed9dc193bd3, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/f46fa0e3d8ee469b87208b9aa85bb9af, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/da6564d5050746f0be4a0e346a8aac5b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/cc8ddcf24f354b99a34233aeb096b928, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/73e6509f1195408789136114d4cb4291, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/ad0ab517ae0e4375b0022c23f106259d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/8f83bcc9394d46ca93c1a6d17cbaf065, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/5f7bb0be225a4c57aaf6a898b17ce3a0, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/751619eb77dd4053950d4bfb8b5996a0, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/c83a614fb8e14db39abd70a599336bbf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/348c6236986e4d98bbb9082a2e2cb8e7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d1e8fedb176142ff8edcdfba2bfecc2b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/32a72f6d08024f3ba6418773e1b1e0bc, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d73219b8b9a84d6d871e0b76cfad9f76, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d33469ea9bc8401d9f0e48db3ae6b6f4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/0e167a519dfc45aebc3d0378d00f4a66, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d7d0e820247a402d8a2981101692d724, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/2b7182d0d82e4a61b7ac9eae31f2e2d2, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/6a71f48dcc6040f59541625139b66d10, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/ca0605c97ca04239a7247f7ad9566309, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/3234212f04774136b9cae13dfeb51430, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/2abab10a3ff6461e9d5d1b028f314edd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/aaf69db593644847bc697f378274a7e7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/aa1d9e27905c4aafaab6bc83e8705665] to archive 2024-11-17T21:26:05,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:26:05,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/c15be5fb8657466abcd37ee0701964a7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/c15be5fb8657466abcd37ee0701964a7 2024-11-17T21:26:05,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/4e07432d038d43b6a59d5f912eea0fcd to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/4e07432d038d43b6a59d5f912eea0fcd 2024-11-17T21:26:05,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/14f7bf4893e24f2490c0eed9dc193bd3 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/14f7bf4893e24f2490c0eed9dc193bd3 2024-11-17T21:26:05,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/f46fa0e3d8ee469b87208b9aa85bb9af to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/f46fa0e3d8ee469b87208b9aa85bb9af 2024-11-17T21:26:05,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/da6564d5050746f0be4a0e346a8aac5b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/da6564d5050746f0be4a0e346a8aac5b 2024-11-17T21:26:05,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/cc8ddcf24f354b99a34233aeb096b928 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/cc8ddcf24f354b99a34233aeb096b928 2024-11-17T21:26:05,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/73e6509f1195408789136114d4cb4291 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/73e6509f1195408789136114d4cb4291 2024-11-17T21:26:05,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/ad0ab517ae0e4375b0022c23f106259d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/ad0ab517ae0e4375b0022c23f106259d 2024-11-17T21:26:05,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/8f83bcc9394d46ca93c1a6d17cbaf065 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/8f83bcc9394d46ca93c1a6d17cbaf065 2024-11-17T21:26:05,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/5f7bb0be225a4c57aaf6a898b17ce3a0 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/5f7bb0be225a4c57aaf6a898b17ce3a0 2024-11-17T21:26:05,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/751619eb77dd4053950d4bfb8b5996a0 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/751619eb77dd4053950d4bfb8b5996a0 2024-11-17T21:26:05,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/c83a614fb8e14db39abd70a599336bbf to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/c83a614fb8e14db39abd70a599336bbf 2024-11-17T21:26:05,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/348c6236986e4d98bbb9082a2e2cb8e7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/348c6236986e4d98bbb9082a2e2cb8e7 2024-11-17T21:26:05,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d1e8fedb176142ff8edcdfba2bfecc2b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d1e8fedb176142ff8edcdfba2bfecc2b 2024-11-17T21:26:05,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/32a72f6d08024f3ba6418773e1b1e0bc to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/32a72f6d08024f3ba6418773e1b1e0bc 2024-11-17T21:26:05,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d73219b8b9a84d6d871e0b76cfad9f76 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d73219b8b9a84d6d871e0b76cfad9f76 2024-11-17T21:26:05,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d33469ea9bc8401d9f0e48db3ae6b6f4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d33469ea9bc8401d9f0e48db3ae6b6f4 2024-11-17T21:26:05,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/0e167a519dfc45aebc3d0378d00f4a66 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/0e167a519dfc45aebc3d0378d00f4a66 2024-11-17T21:26:05,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d7d0e820247a402d8a2981101692d724 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/d7d0e820247a402d8a2981101692d724 2024-11-17T21:26:05,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/2b7182d0d82e4a61b7ac9eae31f2e2d2 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/2b7182d0d82e4a61b7ac9eae31f2e2d2 2024-11-17T21:26:05,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/6a71f48dcc6040f59541625139b66d10 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/6a71f48dcc6040f59541625139b66d10 2024-11-17T21:26:05,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/ca0605c97ca04239a7247f7ad9566309 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/ca0605c97ca04239a7247f7ad9566309 2024-11-17T21:26:05,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/3234212f04774136b9cae13dfeb51430 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/3234212f04774136b9cae13dfeb51430 2024-11-17T21:26:05,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/2abab10a3ff6461e9d5d1b028f314edd to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/2abab10a3ff6461e9d5d1b028f314edd 2024-11-17T21:26:05,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/aaf69db593644847bc697f378274a7e7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/aaf69db593644847bc697f378274a7e7 2024-11-17T21:26:05,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/aa1d9e27905c4aafaab6bc83e8705665 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/aa1d9e27905c4aafaab6bc83e8705665 2024-11-17T21:26:05,458 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/recovered.edits/402.seqid, newMaxSeqId=402, maxSeqId=4 2024-11-17T21:26:05,459 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. 2024-11-17T21:26:05,459 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for c3ee12f1af122429c9b29c87eab606c8: Waiting for close lock at 1731878763681Running coprocessor pre-close hooks at 1731878763681Disabling compacts and flushes for region at 1731878763681Disabling writes for close at 1731878763681Obtaining lock to block concurrent updates at 1731878763682 (+1 ms)Preparing flush snapshotting stores in c3ee12f1af122429c9b29c87eab606c8 at 1731878763682Finished memstore snapshotting TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731878763682Flushing stores of TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8. at 1731878763683 (+1 ms)Flushing c3ee12f1af122429c9b29c87eab606c8/A: creating writer at 1731878763684 (+1 ms)Flushing mob file c3ee12f1af122429c9b29c87eab606c8/A: appending metadata at 1731878763693 (+9 ms)Flushing mob file c3ee12f1af122429c9b29c87eab606c8/A: closing flushed file at 1731878763693Flushing c3ee12f1af122429c9b29c87eab606c8/A: appending metadata at 1731878764110 (+417 ms)Flushing c3ee12f1af122429c9b29c87eab606c8/A: closing flushed file at 1731878764110Flushing c3ee12f1af122429c9b29c87eab606c8/B: creating writer at 1731878764526 (+416 ms)Flushing c3ee12f1af122429c9b29c87eab606c8/B: appending metadata at 1731878764529 (+3 ms)Flushing c3ee12f1af122429c9b29c87eab606c8/B: closing flushed file at 1731878764529Flushing c3ee12f1af122429c9b29c87eab606c8/C: creating writer at 1731878764943 (+414 ms)Flushing c3ee12f1af122429c9b29c87eab606c8/C: appending metadata at 1731878764947 (+4 ms)Flushing c3ee12f1af122429c9b29c87eab606c8/C: closing flushed file at 1731878764947Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12d0a7b6: reopening flushed file at 1731878765363 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c4e9b09: reopening flushed file at 1731878765368 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3530b4e: reopening flushed file at 1731878765373 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for c3ee12f1af122429c9b29c87eab606c8 in 1697ms, sequenceid=399, compaction requested=false at 1731878765379 (+6 ms)Writing region close event to WAL at 1731878765456 (+77 ms)Running coprocessor post-close hooks at 1731878765459 (+3 ms)Closed at 1731878765459 2024-11-17T21:26:05,460 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,461 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=c3ee12f1af122429c9b29c87eab606c8, regionState=CLOSED 2024-11-17T21:26:05,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:26:05,464 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-17T21:26:05,464 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure c3ee12f1af122429c9b29c87eab606c8, server=a313eea8709e,46445,1731878642828 in 1.9350 sec 2024-11-17T21:26:05,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-17T21:26:05,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c3ee12f1af122429c9b29c87eab606c8, UNASSIGN in 1.9380 sec 2024-11-17T21:26:05,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-17T21:26:05,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9410 sec 2024-11-17T21:26:05,470 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878765469"}]},"ts":"1731878765469"} 2024-11-17T21:26:05,471 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T21:26:05,471 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T21:26:05,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9530 sec 2024-11-17T21:26:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T21:26:05,672 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:05,673 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T21:26:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:05,677 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T21:26:05,679 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:05,682 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,686 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/recovered.edits] 2024-11-17T21:26:05,690 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/7f446cfb30b64e28ac6175e2d5600687 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/7f446cfb30b64e28ac6175e2d5600687 2024-11-17T21:26:05,691 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/b8c5061309704e9c8e517f3a3d4f86ce to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/A/b8c5061309704e9c8e517f3a3d4f86ce 2024-11-17T21:26:05,695 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/538e35551f124267ad734dd572aa02f0 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/538e35551f124267ad734dd572aa02f0 2024-11-17T21:26:05,696 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/aedba1a6c6f148fd96a65651885dc98e to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/B/aedba1a6c6f148fd96a65651885dc98e 2024-11-17T21:26:05,698 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/75f5d81fd5c24828927192b7fd8f46c1 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/75f5d81fd5c24828927192b7fd8f46c1 2024-11-17T21:26:05,699 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/ed43fc5c33cf4459a615e00046077465 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/C/ed43fc5c33cf4459a615e00046077465 2024-11-17T21:26:05,701 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/recovered.edits/402.seqid to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8/recovered.edits/402.seqid 2024-11-17T21:26:05,702 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,702 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T21:26:05,702 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T21:26:05,703 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T21:26:05,706 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111713cab619a31a443bbbed87ade4b67636_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111713cab619a31a443bbbed87ade4b67636_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,706 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172c16ce30bcad427db013577b245e3cd5_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172c16ce30bcad427db013577b245e3cd5_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,707 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172c724278cb6f4379a4f2a49606165e61_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172c724278cb6f4379a4f2a49606165e61_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,708 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117339e0b23760e4630a3889205fe36c55e_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117339e0b23760e4630a3889205fe36c55e_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,709 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111741aa6d4b284a411b87cf8d14f8652f87_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111741aa6d4b284a411b87cf8d14f8652f87_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,710 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111743f83c525c6441a68c9f08b8733c640e_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111743f83c525c6441a68c9f08b8733c640e_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,710 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117684d08f5e64042a9b1c7b1191364cc9b_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117684d08f5e64042a9b1c7b1191364cc9b_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,711 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117715728ca199f452dbf9fedb79fd6d1f8_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117715728ca199f452dbf9fedb79fd6d1f8_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,712 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117749688b9346340b7a70700f3b0373c7e_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117749688b9346340b7a70700f3b0373c7e_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,713 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178f623284afd844f2b88cfd8917a268df_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178f623284afd844f2b88cfd8917a268df_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,714 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179ce17fb8d3444600a57b80abbdc5c62a_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179ce17fb8d3444600a57b80abbdc5c62a_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,714 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a551ba985e9b45ef9ce20aa22ffadc56_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a551ba985e9b45ef9ce20aa22ffadc56_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,715 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a6e725b484514fbcb15f8ba583acd639_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a6e725b484514fbcb15f8ba583acd639_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,716 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b7a25c50dadd43a3bb1ba984a33a478c_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b7a25c50dadd43a3bb1ba984a33a478c_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,717 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b7db32072c324ade9b80b7a737678d8a_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b7db32072c324ade9b80b7a737678d8a_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,717 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ba25fb503c1d4d2cabe5ef5edc950e28_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ba25fb503c1d4d2cabe5ef5edc950e28_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,718 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cf91ca16ac0741bea21a332e0e1933a5_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cf91ca16ac0741bea21a332e0e1933a5_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,719 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d45ce956296a4150b653507fa521b12b_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d45ce956296a4150b653507fa521b12b_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,720 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117da69c9d94b494549b109c7263d488ca1_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117da69c9d94b494549b109c7263d488ca1_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,720 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f628693fd5244c42a4d7f61ed0b763b3_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f628693fd5244c42a4d7f61ed0b763b3_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,721 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fd710a43aec04f42a701fda6fda1aca7_c3ee12f1af122429c9b29c87eab606c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fd710a43aec04f42a701fda6fda1aca7_c3ee12f1af122429c9b29c87eab606c8 2024-11-17T21:26:05,722 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T21:26:05,723 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:05,725 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T21:26:05,727 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T21:26:05,728 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:05,728 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T21:26:05,728 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731878765728"}]},"ts":"9223372036854775807"} 2024-11-17T21:26:05,730 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T21:26:05,730 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c3ee12f1af122429c9b29c87eab606c8, NAME => 'TestAcidGuarantees,,1731878730771.c3ee12f1af122429c9b29c87eab606c8.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T21:26:05,730 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T21:26:05,730 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731878765730"}]},"ts":"9223372036854775807"} 2024-11-17T21:26:05,732 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T21:26:05,732 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:05,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-11-17T21:26:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T21:26:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T21:26:05,781 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:05,791 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237 (was 240), OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=521 (was 547), ProcessCount=11 (was 11), AvailableMemoryMB=8150 (was 8322) 2024-11-17T21:26:05,802 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=521, ProcessCount=11, AvailableMemoryMB=8149 2024-11-17T21:26:05,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T21:26:05,804 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T21:26:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:05,806 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T21:26:05,806 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:05,806 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-17T21:26:05,807 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T21:26:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T21:26:05,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742271_1447 (size=959) 2024-11-17T21:26:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T21:26:05,999 WARN [regionserver/a313eea8709e:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-17T21:26:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T21:26:06,217 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249 2024-11-17T21:26:06,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742272_1448 (size=53) 2024-11-17T21:26:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T21:26:06,629 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:26:06,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1e1006bdbe13128ffffb3f217a9e33ec, disabling compactions & flushes 2024-11-17T21:26:06,630 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:06,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:06,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. after waiting 0 ms 2024-11-17T21:26:06,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:06,630 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:06,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1e1006bdbe13128ffffb3f217a9e33ec: Waiting for close lock at 1731878766630Disabling compacts and flushes for region at 1731878766630Disabling writes for close at 1731878766630Writing region close event to WAL at 1731878766630Closed at 1731878766630 2024-11-17T21:26:06,633 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T21:26:06,633 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731878766633"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731878766633"}]},"ts":"1731878766633"} 2024-11-17T21:26:06,636 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T21:26:06,637 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T21:26:06,637 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878766637"}]},"ts":"1731878766637"} 2024-11-17T21:26:06,640 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T21:26:06,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1006bdbe13128ffffb3f217a9e33ec, ASSIGN}] 2024-11-17T21:26:06,641 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1006bdbe13128ffffb3f217a9e33ec, ASSIGN 2024-11-17T21:26:06,642 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1006bdbe13128ffffb3f217a9e33ec, ASSIGN; state=OFFLINE, location=a313eea8709e,46445,1731878642828; forceNewPlan=false, retain=false 2024-11-17T21:26:06,793 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=1e1006bdbe13128ffffb3f217a9e33ec, regionState=OPENING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:26:06,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1006bdbe13128ffffb3f217a9e33ec, ASSIGN because future has completed 2024-11-17T21:26:06,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:26:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T21:26:06,959 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:06,959 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} 2024-11-17T21:26:06,961 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:06,961 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:26:06,961 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:06,961 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:06,964 INFO [StoreOpener-1e1006bdbe13128ffffb3f217a9e33ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:06,966 INFO [StoreOpener-1e1006bdbe13128ffffb3f217a9e33ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e1006bdbe13128ffffb3f217a9e33ec columnFamilyName A 2024-11-17T21:26:06,966 DEBUG [StoreOpener-1e1006bdbe13128ffffb3f217a9e33ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:06,966 INFO [StoreOpener-1e1006bdbe13128ffffb3f217a9e33ec-1 {}] regionserver.HStore(327): Store=1e1006bdbe13128ffffb3f217a9e33ec/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:26:06,967 INFO [StoreOpener-1e1006bdbe13128ffffb3f217a9e33ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:06,968 INFO [StoreOpener-1e1006bdbe13128ffffb3f217a9e33ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e1006bdbe13128ffffb3f217a9e33ec columnFamilyName B 2024-11-17T21:26:06,968 DEBUG [StoreOpener-1e1006bdbe13128ffffb3f217a9e33ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:06,968 INFO [StoreOpener-1e1006bdbe13128ffffb3f217a9e33ec-1 {}] regionserver.HStore(327): Store=1e1006bdbe13128ffffb3f217a9e33ec/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:26:06,968 INFO [StoreOpener-1e1006bdbe13128ffffb3f217a9e33ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:06,970 INFO [StoreOpener-1e1006bdbe13128ffffb3f217a9e33ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e1006bdbe13128ffffb3f217a9e33ec columnFamilyName C 2024-11-17T21:26:06,970 DEBUG [StoreOpener-1e1006bdbe13128ffffb3f217a9e33ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:06,970 INFO [StoreOpener-1e1006bdbe13128ffffb3f217a9e33ec-1 {}] regionserver.HStore(327): Store=1e1006bdbe13128ffffb3f217a9e33ec/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:26:06,970 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:06,971 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:06,971 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:06,972 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:06,972 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:06,973 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T21:26:06,974 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:06,976 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T21:26:06,977 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened 1e1006bdbe13128ffffb3f217a9e33ec; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66622702, jitterRate=-0.007244378328323364}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T21:26:06,977 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:06,978 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for 1e1006bdbe13128ffffb3f217a9e33ec: Running coprocessor pre-open hook at 1731878766962Writing region info on filesystem at 1731878766962Initializing all the Stores at 1731878766964 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878766964Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878766964Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878766964Cleaning up temporary data from old regions at 1731878766972 (+8 ms)Running coprocessor post-open hooks at 1731878766977 (+5 ms)Region opened successfully at 1731878766978 (+1 ms) 2024-11-17T21:26:06,979 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., pid=118, masterSystemTime=1731878766952 2024-11-17T21:26:06,981 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:06,981 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:06,981 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=1e1006bdbe13128ffffb3f217a9e33ec, regionState=OPEN, openSeqNum=2, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:26:06,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:26:06,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-17T21:26:06,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure 1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 in 187 msec 2024-11-17T21:26:06,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-17T21:26:06,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1006bdbe13128ffffb3f217a9e33ec, ASSIGN in 346 msec 2024-11-17T21:26:06,990 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T21:26:06,990 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878766990"}]},"ts":"1731878766990"} 2024-11-17T21:26:06,992 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T21:26:06,992 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T21:26:06,994 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1880 sec 2024-11-17T21:26:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T21:26:07,962 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:07,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a68ca01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:07,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:07,967 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:07,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@595e4adf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:07,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:07,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:07,969 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b7309a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,971 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:07,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:07,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:07,973 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:07,974 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T21:26:07,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is a313eea8709e,43623,1731878641945 2024-11-17T21:26:07,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27f4e550 2024-11-17T21:26:07,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T21:26:07,976 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T21:26:07,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc6145c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,978 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:07,978 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:07,979 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:07,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:07,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:07,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5236501a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:07,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:07,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:07,981 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:07,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@387f4edd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@323c960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:07,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:07,984 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:07,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:07,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:07,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e51b839, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:07,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:07,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:07,986 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:07,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b811422, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bc71f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:07,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:07,988 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ad7a273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:07,989 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f69ddd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab65de2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:07,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:07,990 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79860a30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:07,992 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28648e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b3d1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:07,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:07,993 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4615c50b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:07,994 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:07,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@473450ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337c4d69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,995 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:07,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:07,995 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@320e40ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:07,996 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:07,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4daf2da7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b5c3343, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,998 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:07,998 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:07,998 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:07,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:07,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@640bdb17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:07,999 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:08,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12966532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:08,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92c812, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:08,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:08,001 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:08,001 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9177c56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:08,002 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@161de748, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:08,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a00d5a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:08,003 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:08,003 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:08,004 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79c2ee92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:08,005 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23aac6ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:08,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55f5073a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:08,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:08,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:08,007 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31f62315, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:08,008 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18d9fa64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:08,009 DEBUG [Thread-2024 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:08,009 DEBUG [Thread-2017 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:08,009 DEBUG [Thread-2022 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:08,009 DEBUG [Thread-2011 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:08,009 DEBUG [Thread-2009 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:08,009 DEBUG [Thread-2013 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:08,010 DEBUG [Thread-2026 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:08,010 DEBUG [Thread-2020 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:08,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:08,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:08,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:08,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:08,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:08,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:08,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:08,011 DEBUG [Thread-2015 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:08,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:08,011 DEBUG [Thread-2028 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:08,012 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:08,012 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:08,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:08,013 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:08,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:26:08,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:26:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:26:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:26:08,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-17T21:26:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:26:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:26:08,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T21:26:08,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:26:08,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:26:08,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:26:08,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:08,017 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:08,017 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:08,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2] 2024-11-17T21:26:08,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:08,047 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:08,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:26:08,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/17f927a3c2b444fa91e7a4307ddc97f7 is 50, key is test_row_0/A:col10/1731878768046/Put/seqid=0 2024-11-17T21:26:08,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742273_1449 (size=14341) 2024-11-17T21:26:08,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/17f927a3c2b444fa91e7a4307ddc97f7 2024-11-17T21:26:08,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878828083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878828084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/9f384543906445c3a4dae5b762831b40 is 50, key is test_row_0/B:col10/1731878768046/Put/seqid=0 2024-11-17T21:26:08,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878828091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878828093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742274_1450 (size=12001) 2024-11-17T21:26:08,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/9f384543906445c3a4dae5b762831b40 2024-11-17T21:26:08,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878828095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T21:26:08,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/40ed739b481f4795ad5ca352602d79f7 is 50, key is test_row_0/C:col10/1731878768046/Put/seqid=0 2024-11-17T21:26:08,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742275_1451 (size=12001) 2024-11-17T21:26:08,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T21:26:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:08,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:08,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878828201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878828201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878828201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878828211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878828211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T21:26:08,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:08,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:08,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:08,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:08,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T21:26:08,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878828421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878828421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878828421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878828430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878828430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T21:26:08,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:08,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:08,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:08,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:08,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:08,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/40ed739b481f4795ad5ca352602d79f7 2024-11-17T21:26:08,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/17f927a3c2b444fa91e7a4307ddc97f7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/17f927a3c2b444fa91e7a4307ddc97f7 2024-11-17T21:26:08,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/17f927a3c2b444fa91e7a4307ddc97f7, entries=200, sequenceid=12, filesize=14.0 K 2024-11-17T21:26:08,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/9f384543906445c3a4dae5b762831b40 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/9f384543906445c3a4dae5b762831b40 2024-11-17T21:26:08,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/9f384543906445c3a4dae5b762831b40, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T21:26:08,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/40ed739b481f4795ad5ca352602d79f7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/40ed739b481f4795ad5ca352602d79f7 2024-11-17T21:26:08,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/40ed739b481f4795ad5ca352602d79f7, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T21:26:08,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1e1006bdbe13128ffffb3f217a9e33ec in 496ms, sequenceid=12, compaction requested=false 2024-11-17T21:26:08,543 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T21:26:08,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T21:26:08,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T21:26:08,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:08,660 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T21:26:08,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/5a97dce264054687a2cad48e79d82fd9 is 50, key is test_row_0/A:col10/1731878768091/Put/seqid=0 2024-11-17T21:26:08,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742276_1452 (size=12001) 2024-11-17T21:26:08,669 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/5a97dce264054687a2cad48e79d82fd9 2024-11-17T21:26:08,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/ca2bf1f3ca2941969f60ac93480ca669 is 50, key is test_row_0/B:col10/1731878768091/Put/seqid=0 2024-11-17T21:26:08,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742277_1453 (size=12001) 2024-11-17T21:26:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:08,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:08,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878828772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878828772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878828773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878828774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878828774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878828891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878828891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878828891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878828891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878828891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:09,078 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/ca2bf1f3ca2941969f60ac93480ca669 2024-11-17T21:26:09,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/2bdb7f5eff1c48099543da4dbcecca46 is 50, key is test_row_0/C:col10/1731878768091/Put/seqid=0 2024-11-17T21:26:09,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742278_1454 (size=12001) 2024-11-17T21:26:09,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878829101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:09,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878829101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878829101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:09,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878829101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:09,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:09,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878829101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:09,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T21:26:09,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878829410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:09,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878829411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:09,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:09,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878829411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:09,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878829411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:09,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878829411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:09,486 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/2bdb7f5eff1c48099543da4dbcecca46 2024-11-17T21:26:09,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/5a97dce264054687a2cad48e79d82fd9 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/5a97dce264054687a2cad48e79d82fd9 2024-11-17T21:26:09,492 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/5a97dce264054687a2cad48e79d82fd9, entries=150, sequenceid=37, filesize=11.7 K 2024-11-17T21:26:09,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/ca2bf1f3ca2941969f60ac93480ca669 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/ca2bf1f3ca2941969f60ac93480ca669 2024-11-17T21:26:09,495 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/ca2bf1f3ca2941969f60ac93480ca669, entries=150, sequenceid=37, filesize=11.7 K 2024-11-17T21:26:09,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/2bdb7f5eff1c48099543da4dbcecca46 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/2bdb7f5eff1c48099543da4dbcecca46 2024-11-17T21:26:09,499 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/2bdb7f5eff1c48099543da4dbcecca46, entries=150, sequenceid=37, filesize=11.7 K 2024-11-17T21:26:09,500 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 1e1006bdbe13128ffffb3f217a9e33ec in 839ms, sequenceid=37, compaction requested=false 2024-11-17T21:26:09,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:09,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:09,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-17T21:26:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-17T21:26:09,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-17T21:26:09,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4840 sec 2024-11-17T21:26:09,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4900 sec 2024-11-17T21:26:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:09,925 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:09,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T21:26:09,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/6fbb1b73307745c49593118197adedab is 50, key is test_row_0/A:col10/1731878769925/Put/seqid=0 2024-11-17T21:26:09,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742279_1455 (size=16681) 2024-11-17T21:26:09,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878829952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:09,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878829953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878829957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:09,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878829957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:09,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:09,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878829964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:09,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:09,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:10,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878830071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:10,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:10,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878830071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:10,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:10,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:10,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878830072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:10,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:10,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878830072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878830072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:10,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T21:26:10,161 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:10,161 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-17T21:26:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T21:26:10,163 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:10,163 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:10,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T21:26:10,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878830281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:10,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878830281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:10,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878830281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878830281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:10,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878830281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:10,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T21:26:10,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:10,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:10,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:10,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:10,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:10,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/6fbb1b73307745c49593118197adedab 2024-11-17T21:26:10,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/f4dbcb1ad32a4fbebfec7a8cba48414b is 50, key is test_row_0/B:col10/1731878769925/Put/seqid=0 2024-11-17T21:26:10,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742280_1456 (size=12001) 2024-11-17T21:26:10,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T21:26:10,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:10,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:10,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:10,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:10,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T21:26:10,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878830591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:10,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878830591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:10,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878830591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:10,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878830601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:10,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:10,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878830601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:10,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:10,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:10,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:10,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T21:26:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:10,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:10,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/f4dbcb1ad32a4fbebfec7a8cba48414b 2024-11-17T21:26:10,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/a3b01a3359074b159b2261dfb4beabda is 50, key is test_row_0/C:col10/1731878769925/Put/seqid=0 2024-11-17T21:26:10,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742281_1457 (size=12001) 2024-11-17T21:26:10,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T21:26:10,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:10,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:10,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:10,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:10,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T21:26:10,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T21:26:10,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:10,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:10,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:10,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:10,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:10,976 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T21:26:11,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T21:26:11,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:11,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:11,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:11,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:11,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:11,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878831101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:11,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878831111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:11,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878831111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:11,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878831112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:11,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878831121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:11,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/a3b01a3359074b159b2261dfb4beabda 2024-11-17T21:26:11,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/6fbb1b73307745c49593118197adedab as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/6fbb1b73307745c49593118197adedab 2024-11-17T21:26:11,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/6fbb1b73307745c49593118197adedab, entries=250, sequenceid=51, filesize=16.3 K 2024-11-17T21:26:11,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/f4dbcb1ad32a4fbebfec7a8cba48414b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/f4dbcb1ad32a4fbebfec7a8cba48414b 2024-11-17T21:26:11,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/f4dbcb1ad32a4fbebfec7a8cba48414b, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T21:26:11,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/a3b01a3359074b159b2261dfb4beabda as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/a3b01a3359074b159b2261dfb4beabda 2024-11-17T21:26:11,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/a3b01a3359074b159b2261dfb4beabda, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T21:26:11,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 1e1006bdbe13128ffffb3f217a9e33ec in 1238ms, sequenceid=51, compaction requested=true 2024-11-17T21:26:11,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:11,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:26:11,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:11,163 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:11,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:26:11,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:11,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:26:11,163 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:11,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:11,164 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:11,164 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:11,164 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/A is initiating minor compaction (all files) 2024-11-17T21:26:11,164 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/B is initiating minor compaction (all files) 2024-11-17T21:26:11,164 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/A in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:11,164 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/B in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:11,164 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/17f927a3c2b444fa91e7a4307ddc97f7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/5a97dce264054687a2cad48e79d82fd9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/6fbb1b73307745c49593118197adedab] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=42.0 K 2024-11-17T21:26:11,164 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/9f384543906445c3a4dae5b762831b40, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/ca2bf1f3ca2941969f60ac93480ca669, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/f4dbcb1ad32a4fbebfec7a8cba48414b] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=35.2 K 2024-11-17T21:26:11,165 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f384543906445c3a4dae5b762831b40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731878768016 2024-11-17T21:26:11,165 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17f927a3c2b444fa91e7a4307ddc97f7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731878768016 2024-11-17T21:26:11,165 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a97dce264054687a2cad48e79d82fd9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731878768090 2024-11-17T21:26:11,165 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting ca2bf1f3ca2941969f60ac93480ca669, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731878768090 2024-11-17T21:26:11,165 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fbb1b73307745c49593118197adedab, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878768743 2024-11-17T21:26:11,165 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting f4dbcb1ad32a4fbebfec7a8cba48414b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878768772 2024-11-17T21:26:11,171 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#B#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:11,171 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#A#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:11,171 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/0e3a418f26174608bdaf3e0e205f6557 is 50, key is test_row_0/B:col10/1731878769925/Put/seqid=0 2024-11-17T21:26:11,172 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/b870a63e9d0f4783b6f665fe3bdbe625 is 50, key is test_row_0/A:col10/1731878769925/Put/seqid=0 2024-11-17T21:26:11,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742282_1458 (size=12104) 2024-11-17T21:26:11,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742283_1459 (size=12104) 2024-11-17T21:26:11,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T21:26:11,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:11,231 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T21:26:11,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/43355e3a9ac04524bc964c5266df74a4 is 50, key is test_row_0/A:col10/1731878769933/Put/seqid=0 2024-11-17T21:26:11,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742284_1460 (size=12001) 2024-11-17T21:26:11,239 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/43355e3a9ac04524bc964c5266df74a4 2024-11-17T21:26:11,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/b228388d49e344eea2f76d4aec737f92 is 50, key is test_row_0/B:col10/1731878769933/Put/seqid=0 2024-11-17T21:26:11,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742285_1461 (size=12001) 2024-11-17T21:26:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T21:26:11,582 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/0e3a418f26174608bdaf3e0e205f6557 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/0e3a418f26174608bdaf3e0e205f6557 2024-11-17T21:26:11,585 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/B of 1e1006bdbe13128ffffb3f217a9e33ec into 0e3a418f26174608bdaf3e0e205f6557(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:11,585 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:11,585 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/B, priority=13, startTime=1731878771163; duration=0sec 2024-11-17T21:26:11,585 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:11,585 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:B 2024-11-17T21:26:11,585 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:11,586 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:11,586 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/C is initiating minor compaction (all files) 2024-11-17T21:26:11,586 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/C in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:11,586 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/40ed739b481f4795ad5ca352602d79f7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/2bdb7f5eff1c48099543da4dbcecca46, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/a3b01a3359074b159b2261dfb4beabda] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=35.2 K 2024-11-17T21:26:11,586 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 40ed739b481f4795ad5ca352602d79f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731878768016 2024-11-17T21:26:11,586 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bdb7f5eff1c48099543da4dbcecca46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731878768090 2024-11-17T21:26:11,587 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting a3b01a3359074b159b2261dfb4beabda, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878768772 2024-11-17T21:26:11,588 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/b870a63e9d0f4783b6f665fe3bdbe625 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/b870a63e9d0f4783b6f665fe3bdbe625 2024-11-17T21:26:11,593 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/A of 1e1006bdbe13128ffffb3f217a9e33ec into b870a63e9d0f4783b6f665fe3bdbe625(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:11,594 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:11,594 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/A, priority=13, startTime=1731878771163; duration=0sec 2024-11-17T21:26:11,594 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:11,594 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:A 2024-11-17T21:26:11,595 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#C#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:11,596 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/bf48abf8380a4478953a49ef232d9113 is 50, key is test_row_0/C:col10/1731878769925/Put/seqid=0 2024-11-17T21:26:11,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742286_1462 (size=12104) 2024-11-17T21:26:11,602 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/bf48abf8380a4478953a49ef232d9113 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/bf48abf8380a4478953a49ef232d9113 2024-11-17T21:26:11,605 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/C of 1e1006bdbe13128ffffb3f217a9e33ec into bf48abf8380a4478953a49ef232d9113(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:11,605 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:11,605 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/C, priority=13, startTime=1731878771163; duration=0sec 2024-11-17T21:26:11,605 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:11,605 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:C 2024-11-17T21:26:11,649 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/b228388d49e344eea2f76d4aec737f92 2024-11-17T21:26:11,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/b58db119438a4582bd59a2578ad41e27 is 50, key is test_row_0/C:col10/1731878769933/Put/seqid=0 2024-11-17T21:26:11,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742287_1463 (size=12001) 2024-11-17T21:26:12,059 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/b58db119438a4582bd59a2578ad41e27 2024-11-17T21:26:12,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/43355e3a9ac04524bc964c5266df74a4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/43355e3a9ac04524bc964c5266df74a4 2024-11-17T21:26:12,068 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/43355e3a9ac04524bc964c5266df74a4, entries=150, sequenceid=73, filesize=11.7 K 2024-11-17T21:26:12,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/b228388d49e344eea2f76d4aec737f92 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b228388d49e344eea2f76d4aec737f92 2024-11-17T21:26:12,072 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b228388d49e344eea2f76d4aec737f92, entries=150, sequenceid=73, filesize=11.7 K 2024-11-17T21:26:12,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/b58db119438a4582bd59a2578ad41e27 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/b58db119438a4582bd59a2578ad41e27 2024-11-17T21:26:12,076 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/b58db119438a4582bd59a2578ad41e27, entries=150, sequenceid=73, filesize=11.7 K 2024-11-17T21:26:12,076 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 1e1006bdbe13128ffffb3f217a9e33ec in 845ms, sequenceid=73, compaction requested=false 2024-11-17T21:26:12,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:12,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:12,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-17T21:26:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-17T21:26:12,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-17T21:26:12,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9150 sec 2024-11-17T21:26:12,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.9190 sec 2024-11-17T21:26:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:12,136 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:12,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:26:12,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/31e24a8e816845fcbf9eb20b2783186c is 50, key is test_row_0/A:col10/1731878772135/Put/seqid=0 2024-11-17T21:26:12,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742288_1464 (size=21365) 2024-11-17T21:26:12,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878832175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878832176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878832185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878832186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878832186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878832290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878832291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878832301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878832301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878832301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T21:26:12,311 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:12,311 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-17T21:26:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T21:26:12,313 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:12,314 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:12,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T21:26:12,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T21:26:12,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:12,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:12,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:12,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:12,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:12,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878832501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878832501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878832510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878832510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878832521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/31e24a8e816845fcbf9eb20b2783186c 2024-11-17T21:26:12,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/5e88586c16d5412795c12ef95323d10b is 50, key is test_row_0/B:col10/1731878772135/Put/seqid=0 2024-11-17T21:26:12,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742289_1465 (size=12001) 2024-11-17T21:26:12,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T21:26:12,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:12,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:12,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:12,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:12,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T21:26:12,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T21:26:12,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:12,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:12,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:12,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:12,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:12,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878832821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878832821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878832821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878832831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:12,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878832831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:12,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:12,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:12,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T21:26:12,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:12,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:12,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:12,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:12,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:12,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:12,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T21:26:12,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/5e88586c16d5412795c12ef95323d10b 2024-11-17T21:26:12,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/d234c8913b344a27a40866318545c523 is 50, key is test_row_0/C:col10/1731878772135/Put/seqid=0 2024-11-17T21:26:12,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742290_1466 (size=12001) 2024-11-17T21:26:13,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T21:26:13,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:13,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:13,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:13,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:13,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:13,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T21:26:13,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:13,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:13,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:13,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:13,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:13,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:13,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:13,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878833331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:13,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:13,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:13,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:13,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878833341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:13,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878833341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:13,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878833341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:13,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878833341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:13,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:13,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/d234c8913b344a27a40866318545c523 2024-11-17T21:26:13,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/31e24a8e816845fcbf9eb20b2783186c as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/31e24a8e816845fcbf9eb20b2783186c 2024-11-17T21:26:13,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/31e24a8e816845fcbf9eb20b2783186c, entries=350, sequenceid=87, filesize=20.9 K 2024-11-17T21:26:13,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/5e88586c16d5412795c12ef95323d10b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/5e88586c16d5412795c12ef95323d10b 2024-11-17T21:26:13,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T21:26:13,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:13,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:13,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:13,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:13,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:13,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/5e88586c16d5412795c12ef95323d10b, entries=150, sequenceid=87, filesize=11.7 K 2024-11-17T21:26:13,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/d234c8913b344a27a40866318545c523 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/d234c8913b344a27a40866318545c523 2024-11-17T21:26:13,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/d234c8913b344a27a40866318545c523, entries=150, sequenceid=87, filesize=11.7 K 2024-11-17T21:26:13,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1e1006bdbe13128ffffb3f217a9e33ec in 1252ms, sequenceid=87, compaction requested=true 2024-11-17T21:26:13,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:13,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:26:13,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:13,388 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:13,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:26:13,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:13,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:26:13,388 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:13,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:13,389 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45470 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:13,389 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:13,389 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/A is initiating minor compaction (all files) 2024-11-17T21:26:13,389 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/B is initiating minor compaction (all files) 2024-11-17T21:26:13,389 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/A in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:13,389 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/B in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:13,389 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/b870a63e9d0f4783b6f665fe3bdbe625, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/43355e3a9ac04524bc964c5266df74a4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/31e24a8e816845fcbf9eb20b2783186c] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=44.4 K 2024-11-17T21:26:13,389 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/0e3a418f26174608bdaf3e0e205f6557, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b228388d49e344eea2f76d4aec737f92, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/5e88586c16d5412795c12ef95323d10b] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=35.3 K 2024-11-17T21:26:13,389 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting b870a63e9d0f4783b6f665fe3bdbe625, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878768772 2024-11-17T21:26:13,389 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e3a418f26174608bdaf3e0e205f6557, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878768772 2024-11-17T21:26:13,390 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting b228388d49e344eea2f76d4aec737f92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731878769933 2024-11-17T21:26:13,390 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43355e3a9ac04524bc964c5266df74a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731878769933 2024-11-17T21:26:13,390 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e88586c16d5412795c12ef95323d10b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731878772130 2024-11-17T21:26:13,390 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31e24a8e816845fcbf9eb20b2783186c, keycount=350, bloomtype=ROW, size=20.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731878772124 2024-11-17T21:26:13,396 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#B#compaction#389 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:13,396 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/8a34e3e8762044b09dc4f7895a2b49e6 is 50, key is test_row_0/B:col10/1731878772135/Put/seqid=0 2024-11-17T21:26:13,399 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#A#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:13,399 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/319a6f71e70749b3be70fa68d743b74e is 50, key is test_row_0/A:col10/1731878772135/Put/seqid=0 2024-11-17T21:26:13,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742291_1467 (size=12207) 2024-11-17T21:26:13,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742292_1468 (size=12207) 2024-11-17T21:26:13,408 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/8a34e3e8762044b09dc4f7895a2b49e6 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/8a34e3e8762044b09dc4f7895a2b49e6 2024-11-17T21:26:13,413 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/B of 1e1006bdbe13128ffffb3f217a9e33ec into 8a34e3e8762044b09dc4f7895a2b49e6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:13,413 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:13,413 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/B, priority=13, startTime=1731878773388; duration=0sec 2024-11-17T21:26:13,413 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:13,413 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:B 2024-11-17T21:26:13,413 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:13,414 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:13,414 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/C is initiating minor compaction (all files) 2024-11-17T21:26:13,414 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/C in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:13,414 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/bf48abf8380a4478953a49ef232d9113, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/b58db119438a4582bd59a2578ad41e27, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/d234c8913b344a27a40866318545c523] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=35.3 K 2024-11-17T21:26:13,414 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting bf48abf8380a4478953a49ef232d9113, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731878768772 2024-11-17T21:26:13,415 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting b58db119438a4582bd59a2578ad41e27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731878769933 2024-11-17T21:26:13,415 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d234c8913b344a27a40866318545c523, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731878772130 2024-11-17T21:26:13,421 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#C#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:13,421 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/254464c7ef7c4b14b62cea08b91e0be6 is 50, key is test_row_0/C:col10/1731878772135/Put/seqid=0 2024-11-17T21:26:13,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742293_1469 (size=12207) 2024-11-17T21:26:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T21:26:13,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T21:26:13,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:13,536 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T21:26:13,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/72060dca48d94133a1d043f82415e1b4 is 50, key is test_row_0/A:col10/1731878772184/Put/seqid=0 2024-11-17T21:26:13,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742294_1470 (size=12001) 2024-11-17T21:26:13,809 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/319a6f71e70749b3be70fa68d743b74e as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/319a6f71e70749b3be70fa68d743b74e 2024-11-17T21:26:13,812 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/A of 1e1006bdbe13128ffffb3f217a9e33ec into 319a6f71e70749b3be70fa68d743b74e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:13,812 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:13,812 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/A, priority=13, startTime=1731878773388; duration=0sec 2024-11-17T21:26:13,812 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:13,812 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:A 2024-11-17T21:26:13,827 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/254464c7ef7c4b14b62cea08b91e0be6 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/254464c7ef7c4b14b62cea08b91e0be6 2024-11-17T21:26:13,831 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/C of 1e1006bdbe13128ffffb3f217a9e33ec into 254464c7ef7c4b14b62cea08b91e0be6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:13,831 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:13,831 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/C, priority=13, startTime=1731878773388; duration=0sec 2024-11-17T21:26:13,831 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:13,831 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:C 2024-11-17T21:26:13,949 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/72060dca48d94133a1d043f82415e1b4 2024-11-17T21:26:13,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/cce222e7613f48b997d7c34ab8bae5c6 is 50, key is test_row_0/B:col10/1731878772184/Put/seqid=0 2024-11-17T21:26:13,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742295_1471 (size=12001) 2024-11-17T21:26:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:14,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:14,359 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/cce222e7613f48b997d7c34ab8bae5c6 2024-11-17T21:26:14,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/43cdc12ba69c4b11bbe983abb6f94b3a is 50, key is test_row_0/C:col10/1731878772184/Put/seqid=0 2024-11-17T21:26:14,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742296_1472 (size=12001) 2024-11-17T21:26:14,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878834362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:14,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:14,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878834363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:14,370 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_0' in 1e1006bdbe13128ffffb3f217a9e33ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:14,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878834367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:14,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:14,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878834369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:14,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878834376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T21:26:14,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878834481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:14,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878834481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:14,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878834481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:14,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878834491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:14,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878834690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878834691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:14,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878834691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:14,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878834701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:14,769 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/43cdc12ba69c4b11bbe983abb6f94b3a 2024-11-17T21:26:14,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/72060dca48d94133a1d043f82415e1b4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/72060dca48d94133a1d043f82415e1b4 2024-11-17T21:26:14,775 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/72060dca48d94133a1d043f82415e1b4, entries=150, sequenceid=113, filesize=11.7 K 2024-11-17T21:26:14,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/cce222e7613f48b997d7c34ab8bae5c6 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/cce222e7613f48b997d7c34ab8bae5c6 2024-11-17T21:26:14,778 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/cce222e7613f48b997d7c34ab8bae5c6, entries=150, sequenceid=113, filesize=11.7 K 2024-11-17T21:26:14,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/43cdc12ba69c4b11bbe983abb6f94b3a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/43cdc12ba69c4b11bbe983abb6f94b3a 2024-11-17T21:26:14,781 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/43cdc12ba69c4b11bbe983abb6f94b3a, entries=150, sequenceid=113, filesize=11.7 K 2024-11-17T21:26:14,782 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1e1006bdbe13128ffffb3f217a9e33ec in 1247ms, sequenceid=113, compaction requested=false 2024-11-17T21:26:14,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:14,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:14,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-17T21:26:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-17T21:26:14,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-17T21:26:14,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4690 sec 2024-11-17T21:26:14,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.4740 sec 2024-11-17T21:26:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:15,003 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:15,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T21:26:15,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/cadf4cb042be4695a30353c72a7b466e is 50, key is test_row_0/A:col10/1731878775003/Put/seqid=0 2024-11-17T21:26:15,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742297_1473 (size=16681) 2024-11-17T21:26:15,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878835034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:15,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878835038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:15,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878835042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:15,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878835043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:15,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878835151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:15,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878835151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:15,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878835151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:15,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878835160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:15,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878835362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:15,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878835362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:15,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878835372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:15,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878835373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:15,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/cadf4cb042be4695a30353c72a7b466e 2024-11-17T21:26:15,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/9213b5209a5f4ab59e67a9b33b1a52ce is 50, key is test_row_0/B:col10/1731878775003/Put/seqid=0 2024-11-17T21:26:15,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742298_1474 (size=12001) 2024-11-17T21:26:15,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878835681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:15,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878835681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:15,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878835691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:15,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878835691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:15,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/9213b5209a5f4ab59e67a9b33b1a52ce 2024-11-17T21:26:15,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/7068c193f8ae4c99b3ee14c7cc955b28 is 50, key is test_row_0/C:col10/1731878775003/Put/seqid=0 2024-11-17T21:26:15,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742299_1475 (size=12001) 2024-11-17T21:26:16,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878836191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:16,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:16,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:16,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:16,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878836201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:16,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:16,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:16,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:16,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878836210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:16,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878836211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:16,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:16,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:16,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:16,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:16,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:16,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:16,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/7068c193f8ae4c99b3ee14c7cc955b28 2024-11-17T21:26:16,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/cadf4cb042be4695a30353c72a7b466e as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/cadf4cb042be4695a30353c72a7b466e 2024-11-17T21:26:16,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/cadf4cb042be4695a30353c72a7b466e, entries=250, sequenceid=128, filesize=16.3 K 2024-11-17T21:26:16,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/9213b5209a5f4ab59e67a9b33b1a52ce as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/9213b5209a5f4ab59e67a9b33b1a52ce 2024-11-17T21:26:16,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/9213b5209a5f4ab59e67a9b33b1a52ce, entries=150, sequenceid=128, filesize=11.7 K 2024-11-17T21:26:16,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/7068c193f8ae4c99b3ee14c7cc955b28 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/7068c193f8ae4c99b3ee14c7cc955b28 2024-11-17T21:26:16,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/7068c193f8ae4c99b3ee14c7cc955b28, entries=150, sequenceid=128, filesize=11.7 K 2024-11-17T21:26:16,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1e1006bdbe13128ffffb3f217a9e33ec in 1259ms, sequenceid=128, compaction requested=true 2024-11-17T21:26:16,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:16,262 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:16,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:26:16,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:16,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:26:16,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:16,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:26:16,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T21:26:16,263 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:16,264 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:16,264 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/A is initiating minor compaction (all files) 2024-11-17T21:26:16,264 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:16,265 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/B is initiating minor compaction (all files) 2024-11-17T21:26:16,265 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/A in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:16,265 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/B in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:16,265 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/8a34e3e8762044b09dc4f7895a2b49e6, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/cce222e7613f48b997d7c34ab8bae5c6, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/9213b5209a5f4ab59e67a9b33b1a52ce] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=35.4 K 2024-11-17T21:26:16,265 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/319a6f71e70749b3be70fa68d743b74e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/72060dca48d94133a1d043f82415e1b4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/cadf4cb042be4695a30353c72a7b466e] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=39.9 K 2024-11-17T21:26:16,265 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 319a6f71e70749b3be70fa68d743b74e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731878772130 2024-11-17T21:26:16,265 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a34e3e8762044b09dc4f7895a2b49e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731878772130 2024-11-17T21:26:16,265 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 72060dca48d94133a1d043f82415e1b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731878772183 2024-11-17T21:26:16,265 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting cce222e7613f48b997d7c34ab8bae5c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731878772183 2024-11-17T21:26:16,266 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9213b5209a5f4ab59e67a9b33b1a52ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731878774367 2024-11-17T21:26:16,266 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting cadf4cb042be4695a30353c72a7b466e, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731878774362 2024-11-17T21:26:16,275 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#B#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:16,275 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/6d9fe119c0c94d78bbc367bafcb417ca is 50, key is test_row_0/B:col10/1731878775003/Put/seqid=0 2024-11-17T21:26:16,277 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#A#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:16,277 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/e09a3b195d2b4b55a80adcfde3674c0a is 50, key is test_row_0/A:col10/1731878775003/Put/seqid=0 2024-11-17T21:26:16,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742300_1476 (size=12309) 2024-11-17T21:26:16,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742301_1477 (size=12309) 2024-11-17T21:26:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:16,384 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:16,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T21:26:16,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/d4ee5e50444c45a3a04bd6070cc0ebe4 is 50, key is test_row_0/A:col10/1731878776383/Put/seqid=0 2024-11-17T21:26:16,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742302_1478 (size=14541) 2024-11-17T21:26:16,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878836444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:16,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:16,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:16,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T21:26:16,471 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:16,471 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-17T21:26:16,473 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T21:26:16,473 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:16,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:16,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878836551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:16,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:16,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:16,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T21:26:16,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T21:26:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:16,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:16,687 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/6d9fe119c0c94d78bbc367bafcb417ca as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/6d9fe119c0c94d78bbc367bafcb417ca 2024-11-17T21:26:16,715 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/B of 1e1006bdbe13128ffffb3f217a9e33ec into 6d9fe119c0c94d78bbc367bafcb417ca(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:16,715 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:16,715 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/B, priority=13, startTime=1731878776262; duration=0sec 2024-11-17T21:26:16,715 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:16,715 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:B 2024-11-17T21:26:16,715 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:16,716 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:16,716 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/C is initiating minor compaction (all files) 2024-11-17T21:26:16,716 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/C in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:16,717 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/254464c7ef7c4b14b62cea08b91e0be6, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/43cdc12ba69c4b11bbe983abb6f94b3a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/7068c193f8ae4c99b3ee14c7cc955b28] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=35.4 K 2024-11-17T21:26:16,717 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 254464c7ef7c4b14b62cea08b91e0be6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731878772130 2024-11-17T21:26:16,717 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43cdc12ba69c4b11bbe983abb6f94b3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731878772183 2024-11-17T21:26:16,717 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7068c193f8ae4c99b3ee14c7cc955b28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731878774367 2024-11-17T21:26:16,726 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/e09a3b195d2b4b55a80adcfde3674c0a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/e09a3b195d2b4b55a80adcfde3674c0a 2024-11-17T21:26:16,734 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#C#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:16,735 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/39ec304604194e53b7a1b5f07329bd49 is 50, key is test_row_0/C:col10/1731878775003/Put/seqid=0 2024-11-17T21:26:16,738 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/A of 1e1006bdbe13128ffffb3f217a9e33ec into e09a3b195d2b4b55a80adcfde3674c0a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:16,738 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:16,738 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/A, priority=13, startTime=1731878776262; duration=0sec 2024-11-17T21:26:16,738 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:16,738 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:A 2024-11-17T21:26:16,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742303_1479 (size=12309) 2024-11-17T21:26:16,760 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/39ec304604194e53b7a1b5f07329bd49 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/39ec304604194e53b7a1b5f07329bd49 2024-11-17T21:26:16,771 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/C of 1e1006bdbe13128ffffb3f217a9e33ec into 39ec304604194e53b7a1b5f07329bd49(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:16,771 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:16,771 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/C, priority=13, startTime=1731878776262; duration=0sec 2024-11-17T21:26:16,771 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:16,771 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:C 2024-11-17T21:26:16,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878836771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:16,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T21:26:16,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:16,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:16,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:16,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:16,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T21:26:16,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/d4ee5e50444c45a3a04bd6070cc0ebe4 2024-11-17T21:26:16,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/60c30f129a414912ac0bd820534c6fbf is 50, key is test_row_0/B:col10/1731878776383/Put/seqid=0 2024-11-17T21:26:16,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742304_1480 (size=12151) 2024-11-17T21:26:16,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T21:26:16,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:16,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:16,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:16,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:16,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:17,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T21:26:17,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:17,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:17,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:17,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:17,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:17,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878837091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:17,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:17,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:17,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T21:26:17,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/60c30f129a414912ac0bd820534c6fbf 2024-11-17T21:26:17,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878837210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:17,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878837211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:17,218 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_0' in 1e1006bdbe13128ffffb3f217a9e33ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:17,219 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in 1e1006bdbe13128ffffb3f217a9e33ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:17,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/841db4ef4e56493f979720ac3688f2c8 is 50, key is test_row_0/C:col10/1731878776383/Put/seqid=0 2024-11-17T21:26:17,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:17,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878837221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878837221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:17,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742305_1481 (size=12151) 2024-11-17T21:26:17,229 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_2' in 1e1006bdbe13128ffffb3f217a9e33ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:17,229 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in 1e1006bdbe13128ffffb3f217a9e33ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:17,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:17,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:17,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:17,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:17,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:17,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:17,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T21:26:17,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:17,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:17,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:17,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:17,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:17,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T21:26:17,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:17,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:17,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:17,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:17,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:17,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T21:26:17,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:17,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:17,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:17,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:17,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:17,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878837601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:17,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T21:26:17,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/841db4ef4e56493f979720ac3688f2c8 2024-11-17T21:26:17,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/d4ee5e50444c45a3a04bd6070cc0ebe4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/d4ee5e50444c45a3a04bd6070cc0ebe4 2024-11-17T21:26:17,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/d4ee5e50444c45a3a04bd6070cc0ebe4, entries=200, sequenceid=152, filesize=14.2 K 2024-11-17T21:26:17,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/60c30f129a414912ac0bd820534c6fbf as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/60c30f129a414912ac0bd820534c6fbf 2024-11-17T21:26:17,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/60c30f129a414912ac0bd820534c6fbf, entries=150, sequenceid=152, filesize=11.9 K 2024-11-17T21:26:17,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/841db4ef4e56493f979720ac3688f2c8 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/841db4ef4e56493f979720ac3688f2c8 2024-11-17T21:26:17,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/841db4ef4e56493f979720ac3688f2c8, entries=150, sequenceid=152, filesize=11.9 K 2024-11-17T21:26:17,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1e1006bdbe13128ffffb3f217a9e33ec in 1267ms, sequenceid=152, compaction requested=false 2024-11-17T21:26:17,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:17,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T21:26:17,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:17,698 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T21:26:17,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/d2f69149038a40d9a5f32779a4c62229 is 50, key is test_row_0/A:col10/1731878776442/Put/seqid=0 2024-11-17T21:26:17,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742306_1482 (size=12151) 2024-11-17T21:26:17,711 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/d2f69149038a40d9a5f32779a4c62229 2024-11-17T21:26:17,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/26e954e4cb3a41308ba7102bec0640ae is 50, key is test_row_0/B:col10/1731878776442/Put/seqid=0 2024-11-17T21:26:17,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742307_1483 (size=12151) 2024-11-17T21:26:18,130 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/26e954e4cb3a41308ba7102bec0640ae 2024-11-17T21:26:18,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/4a76ae085aec4aad91de82ce72e476f4 is 50, key is test_row_0/C:col10/1731878776442/Put/seqid=0 2024-11-17T21:26:18,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742308_1484 (size=12151) 2024-11-17T21:26:18,159 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/4a76ae085aec4aad91de82ce72e476f4 2024-11-17T21:26:18,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/d2f69149038a40d9a5f32779a4c62229 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/d2f69149038a40d9a5f32779a4c62229 2024-11-17T21:26:18,168 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/d2f69149038a40d9a5f32779a4c62229, entries=150, sequenceid=167, filesize=11.9 K 2024-11-17T21:26:18,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/26e954e4cb3a41308ba7102bec0640ae as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/26e954e4cb3a41308ba7102bec0640ae 2024-11-17T21:26:18,173 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/26e954e4cb3a41308ba7102bec0640ae, entries=150, sequenceid=167, filesize=11.9 K 2024-11-17T21:26:18,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/4a76ae085aec4aad91de82ce72e476f4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/4a76ae085aec4aad91de82ce72e476f4 2024-11-17T21:26:18,181 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/4a76ae085aec4aad91de82ce72e476f4, entries=150, sequenceid=167, filesize=11.9 K 2024-11-17T21:26:18,181 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 1e1006bdbe13128ffffb3f217a9e33ec in 483ms, sequenceid=167, compaction requested=true 2024-11-17T21:26:18,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:18,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:18,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-17T21:26:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-17T21:26:18,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-17T21:26:18,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7100 sec 2024-11-17T21:26:18,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.7140 sec 2024-11-17T21:26:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T21:26:18,621 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:18,622 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-17T21:26:18,623 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T21:26:18,624 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:18,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:18,664 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:18,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:26:18,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/bdc65db635504ce2b70109b81869e3e0 is 50, key is test_row_0/A:col10/1731878778663/Put/seqid=0 2024-11-17T21:26:18,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742309_1485 (size=14541) 2024-11-17T21:26:18,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/bdc65db635504ce2b70109b81869e3e0 2024-11-17T21:26:18,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/3d65950e553d4fa3a9eb7c69d6e3e64a is 50, key is test_row_0/B:col10/1731878778663/Put/seqid=0 2024-11-17T21:26:18,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742310_1486 (size=12151) 2024-11-17T21:26:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T21:26:18,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T21:26:18,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:18,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:18,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:18,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:18,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:18,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878838881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:18,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T21:26:18,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:18,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:18,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:18,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:18,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T21:26:19,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878839001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:19,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:19,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T21:26:19,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:19,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/3d65950e553d4fa3a9eb7c69d6e3e64a 2024-11-17T21:26:19,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/b789807916ea414d9e9bb189de35f6ab is 50, key is test_row_0/C:col10/1731878778663/Put/seqid=0 2024-11-17T21:26:19,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742311_1487 (size=12151) 2024-11-17T21:26:19,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/b789807916ea414d9e9bb189de35f6ab 2024-11-17T21:26:19,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/bdc65db635504ce2b70109b81869e3e0 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/bdc65db635504ce2b70109b81869e3e0 2024-11-17T21:26:19,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/bdc65db635504ce2b70109b81869e3e0, entries=200, sequenceid=178, filesize=14.2 K 2024-11-17T21:26:19,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/3d65950e553d4fa3a9eb7c69d6e3e64a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/3d65950e553d4fa3a9eb7c69d6e3e64a 2024-11-17T21:26:19,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/3d65950e553d4fa3a9eb7c69d6e3e64a, entries=150, sequenceid=178, filesize=11.9 K 2024-11-17T21:26:19,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/b789807916ea414d9e9bb189de35f6ab as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/b789807916ea414d9e9bb189de35f6ab 2024-11-17T21:26:19,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/b789807916ea414d9e9bb189de35f6ab, entries=150, sequenceid=178, filesize=11.9 K 2024-11-17T21:26:19,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1e1006bdbe13128ffffb3f217a9e33ec in 478ms, sequenceid=178, compaction requested=true 2024-11-17T21:26:19,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:19,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:26:19,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:19,143 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:26:19,143 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:26:19,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:26:19,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:19,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:26:19,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:19,144 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:26:19,145 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/B is initiating minor compaction (all files) 2024-11-17T21:26:19,145 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/B in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:19,145 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/6d9fe119c0c94d78bbc367bafcb417ca, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/60c30f129a414912ac0bd820534c6fbf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/26e954e4cb3a41308ba7102bec0640ae, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/3d65950e553d4fa3a9eb7c69d6e3e64a] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=47.6 K 2024-11-17T21:26:19,145 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53542 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:26:19,145 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/A is initiating minor compaction (all files) 2024-11-17T21:26:19,145 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/A in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:19,145 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/e09a3b195d2b4b55a80adcfde3674c0a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/d4ee5e50444c45a3a04bd6070cc0ebe4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/d2f69149038a40d9a5f32779a4c62229, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/bdc65db635504ce2b70109b81869e3e0] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=52.3 K 2024-11-17T21:26:19,145 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d9fe119c0c94d78bbc367bafcb417ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731878774367 2024-11-17T21:26:19,147 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting e09a3b195d2b4b55a80adcfde3674c0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731878774367 2024-11-17T21:26:19,148 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 60c30f129a414912ac0bd820534c6fbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731878775033 2024-11-17T21:26:19,148 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4ee5e50444c45a3a04bd6070cc0ebe4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731878775033 2024-11-17T21:26:19,148 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 26e954e4cb3a41308ba7102bec0640ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731878776413 2024-11-17T21:26:19,148 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2f69149038a40d9a5f32779a4c62229, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731878776413 2024-11-17T21:26:19,148 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d65950e553d4fa3a9eb7c69d6e3e64a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731878778639 2024-11-17T21:26:19,149 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdc65db635504ce2b70109b81869e3e0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731878778639 2024-11-17T21:26:19,167 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#A#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:19,167 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/37b18980a1ab4233869e9695808d06e6 is 50, key is test_row_0/A:col10/1731878778663/Put/seqid=0 2024-11-17T21:26:19,175 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:19,175 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/d48f324a8c4e415b9253f4afb9de8f58 is 50, key is test_row_0/B:col10/1731878778663/Put/seqid=0 2024-11-17T21:26:19,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742312_1488 (size=12595) 2024-11-17T21:26:19,203 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/37b18980a1ab4233869e9695808d06e6 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/37b18980a1ab4233869e9695808d06e6 2024-11-17T21:26:19,207 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/A of 1e1006bdbe13128ffffb3f217a9e33ec into 37b18980a1ab4233869e9695808d06e6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:19,207 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:19,207 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/A, priority=12, startTime=1731878779143; duration=0sec 2024-11-17T21:26:19,207 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:19,207 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:A 2024-11-17T21:26:19,207 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:26:19,211 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:26:19,211 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/C is initiating minor compaction (all files) 2024-11-17T21:26:19,211 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/C in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:19,211 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/39ec304604194e53b7a1b5f07329bd49, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/841db4ef4e56493f979720ac3688f2c8, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/4a76ae085aec4aad91de82ce72e476f4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/b789807916ea414d9e9bb189de35f6ab] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=47.6 K 2024-11-17T21:26:19,211 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39ec304604194e53b7a1b5f07329bd49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731878774367 2024-11-17T21:26:19,211 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 841db4ef4e56493f979720ac3688f2c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731878775033 2024-11-17T21:26:19,212 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a76ae085aec4aad91de82ce72e476f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731878776413 2024-11-17T21:26:19,212 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting b789807916ea414d9e9bb189de35f6ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731878778639 2024-11-17T21:26:19,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742313_1489 (size=12595) 2024-11-17T21:26:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:19,216 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:19,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T21:26:19,220 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#C#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:19,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/6694a53bc4894f2ab5af17349abce7ca is 50, key is test_row_0/A:col10/1731878779215/Put/seqid=0 2024-11-17T21:26:19,221 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/54732f2c8d8f40e79aba7443dd6c27e1 is 50, key is test_row_0/C:col10/1731878778663/Put/seqid=0 2024-11-17T21:26:19,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742315_1491 (size=12595) 2024-11-17T21:26:19,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742314_1490 (size=14541) 2024-11-17T21:26:19,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T21:26:19,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:19,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:19,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:19,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T21:26:19,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878839248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:19,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878839249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:19,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878839250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:19,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878839251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:19,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878839251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:19,261 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_2' in 1e1006bdbe13128ffffb3f217a9e33ec of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:19,261 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in 1e1006bdbe13128ffffb3f217a9e33ec of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:19,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878839361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:19,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:19,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878839370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:19,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:19,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878839371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:19,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:19,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:19,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T21:26:19,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:19,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:19,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:19,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T21:26:19,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:19,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:19,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:19,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878839581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:19,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878839581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:19,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878839582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:19,617 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/d48f324a8c4e415b9253f4afb9de8f58 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/d48f324a8c4e415b9253f4afb9de8f58 2024-11-17T21:26:19,621 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/B of 1e1006bdbe13128ffffb3f217a9e33ec into d48f324a8c4e415b9253f4afb9de8f58(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:19,621 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:19,621 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/B, priority=12, startTime=1731878779143; duration=0sec 2024-11-17T21:26:19,621 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:19,621 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:B 2024-11-17T21:26:19,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/6694a53bc4894f2ab5af17349abce7ca 2024-11-17T21:26:19,634 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/54732f2c8d8f40e79aba7443dd6c27e1 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/54732f2c8d8f40e79aba7443dd6c27e1 2024-11-17T21:26:19,638 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/C of 1e1006bdbe13128ffffb3f217a9e33ec into 54732f2c8d8f40e79aba7443dd6c27e1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:19,638 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:19,638 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/C, priority=12, startTime=1731878779144; duration=0sec 2024-11-17T21:26:19,638 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:19,638 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:C 2024-11-17T21:26:19,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/121a4eefd8c641bbba9a31f4ba8c7363 is 50, key is test_row_0/B:col10/1731878779215/Put/seqid=0 2024-11-17T21:26:19,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742316_1492 (size=12151) 2024-11-17T21:26:19,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T21:26:19,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:19,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:19,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:19,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T21:26:19,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T21:26:19,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:19,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:19,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:19,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:19,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878839891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:19,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878839901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878839901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:19,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:19,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:20,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T21:26:20,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:20,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:20,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:20,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:20,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:20,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/121a4eefd8c641bbba9a31f4ba8c7363 2024-11-17T21:26:20,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/3359c9b1b6d84b58803024a47a72b895 is 50, key is test_row_0/C:col10/1731878779215/Put/seqid=0 2024-11-17T21:26:20,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742317_1493 (size=12151) 2024-11-17T21:26:20,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T21:26:20,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:20,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:20,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:20,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:20,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:20,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T21:26:20,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:20,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:20,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:20,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:20,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:20,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878840412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:20,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878840412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:20,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878840412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:20,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/3359c9b1b6d84b58803024a47a72b895 2024-11-17T21:26:20,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T21:26:20,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:20,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:20,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:20,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:20,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:20,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/6694a53bc4894f2ab5af17349abce7ca as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/6694a53bc4894f2ab5af17349abce7ca 2024-11-17T21:26:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:20,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/6694a53bc4894f2ab5af17349abce7ca, entries=200, sequenceid=205, filesize=14.2 K 2024-11-17T21:26:20,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/121a4eefd8c641bbba9a31f4ba8c7363 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/121a4eefd8c641bbba9a31f4ba8c7363 2024-11-17T21:26:20,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/121a4eefd8c641bbba9a31f4ba8c7363, entries=150, sequenceid=205, filesize=11.9 K 2024-11-17T21:26:20,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/3359c9b1b6d84b58803024a47a72b895 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/3359c9b1b6d84b58803024a47a72b895 2024-11-17T21:26:20,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/3359c9b1b6d84b58803024a47a72b895, entries=150, sequenceid=205, filesize=11.9 K 2024-11-17T21:26:20,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1e1006bdbe13128ffffb3f217a9e33ec in 1267ms, sequenceid=205, compaction requested=false 2024-11-17T21:26:20,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:20,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T21:26:20,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:20,621 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-17T21:26:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/01d93c30e5ef42dcaf35190f098b28a5 is 50, key is test_row_0/A:col10/1731878779248/Put/seqid=0 2024-11-17T21:26:20,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742318_1494 (size=12151) 2024-11-17T21:26:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T21:26:21,031 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/01d93c30e5ef42dcaf35190f098b28a5 2024-11-17T21:26:21,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/e7b7ad3d46de49e987acf8305f08c3bd is 50, key is test_row_0/B:col10/1731878779248/Put/seqid=0 2024-11-17T21:26:21,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742319_1495 (size=12151) 2024-11-17T21:26:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:21,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:21,440 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/e7b7ad3d46de49e987acf8305f08c3bd 2024-11-17T21:26:21,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/c856148a5d6e4524abb63ac15fa55d29 is 50, key is test_row_0/C:col10/1731878779248/Put/seqid=0 2024-11-17T21:26:21,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742320_1496 (size=12151) 2024-11-17T21:26:21,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878841508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:21,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:21,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878841523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:21,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878841523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:21,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878841631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:21,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878841641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:21,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878841641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:21,849 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/c856148a5d6e4524abb63ac15fa55d29 2024-11-17T21:26:21,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/01d93c30e5ef42dcaf35190f098b28a5 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/01d93c30e5ef42dcaf35190f098b28a5 2024-11-17T21:26:21,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878841851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:21,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:21,862 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/01d93c30e5ef42dcaf35190f098b28a5, entries=150, sequenceid=217, filesize=11.9 K 2024-11-17T21:26:21,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/e7b7ad3d46de49e987acf8305f08c3bd as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/e7b7ad3d46de49e987acf8305f08c3bd 2024-11-17T21:26:21,866 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/e7b7ad3d46de49e987acf8305f08c3bd, entries=150, sequenceid=217, filesize=11.9 K 2024-11-17T21:26:21,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/c856148a5d6e4524abb63ac15fa55d29 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/c856148a5d6e4524abb63ac15fa55d29 2024-11-17T21:26:21,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878841861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:21,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878841861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:21,870 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/c856148a5d6e4524abb63ac15fa55d29, entries=150, sequenceid=217, filesize=11.9 K 2024-11-17T21:26:21,870 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 1e1006bdbe13128ffffb3f217a9e33ec in 1249ms, sequenceid=217, compaction requested=true 2024-11-17T21:26:21,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:21,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:21,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-17T21:26:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-17T21:26:21,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-17T21:26:21,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2480 sec 2024-11-17T21:26:21,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 3.2520 sec 2024-11-17T21:26:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:22,169 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:22,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-17T21:26:22,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/f3f544e709e3465ebaf2fcaaf6ce7156 is 50, key is test_row_0/A:col10/1731878781456/Put/seqid=0 2024-11-17T21:26:22,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742321_1497 (size=14541) 2024-11-17T21:26:22,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878842179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:22,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878842188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:22,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878842191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:22,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878842301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:22,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878842301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:22,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878842301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:22,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878842511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:22,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878842511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:22,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878842521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:22,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/f3f544e709e3465ebaf2fcaaf6ce7156 2024-11-17T21:26:22,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/0258aba18a314b749c6b36e288f6b31f is 50, key is test_row_0/B:col10/1731878781456/Put/seqid=0 2024-11-17T21:26:22,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742322_1498 (size=12151) 2024-11-17T21:26:22,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/0258aba18a314b749c6b36e288f6b31f 2024-11-17T21:26:22,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/425d999521ac4b38aed2f1d0412884fc is 50, key is test_row_0/C:col10/1731878781456/Put/seqid=0 2024-11-17T21:26:22,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742323_1499 (size=12151) 2024-11-17T21:26:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T21:26:22,801 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:22,802 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-17T21:26:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T21:26:22,803 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:22,804 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:22,804 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:22,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878842820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:22,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878842820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:22,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878842831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T21:26:22,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T21:26:22,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:22,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:22,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:22,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:22,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:23,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/425d999521ac4b38aed2f1d0412884fc 2024-11-17T21:26:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/f3f544e709e3465ebaf2fcaaf6ce7156 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/f3f544e709e3465ebaf2fcaaf6ce7156 2024-11-17T21:26:23,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/f3f544e709e3465ebaf2fcaaf6ce7156, entries=200, sequenceid=245, filesize=14.2 K 2024-11-17T21:26:23,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/0258aba18a314b749c6b36e288f6b31f as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/0258aba18a314b749c6b36e288f6b31f 2024-11-17T21:26:23,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/0258aba18a314b749c6b36e288f6b31f, entries=150, sequenceid=245, filesize=11.9 K 2024-11-17T21:26:23,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/425d999521ac4b38aed2f1d0412884fc as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/425d999521ac4b38aed2f1d0412884fc 2024-11-17T21:26:23,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/425d999521ac4b38aed2f1d0412884fc, entries=150, sequenceid=245, filesize=11.9 K 2024-11-17T21:26:23,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 1e1006bdbe13128ffffb3f217a9e33ec in 861ms, sequenceid=245, compaction requested=true 2024-11-17T21:26:23,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:23,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:26:23,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:23,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:26:23,031 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:26:23,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:23,031 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:26:23,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:26:23,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:23,033 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:26:23,033 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:26:23,033 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/A is initiating minor compaction (all files) 2024-11-17T21:26:23,033 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/B is initiating minor compaction (all files) 2024-11-17T21:26:23,033 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/A in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:23,033 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/B in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:23,033 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/d48f324a8c4e415b9253f4afb9de8f58, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/121a4eefd8c641bbba9a31f4ba8c7363, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/e7b7ad3d46de49e987acf8305f08c3bd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/0258aba18a314b749c6b36e288f6b31f] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=47.9 K 2024-11-17T21:26:23,033 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/37b18980a1ab4233869e9695808d06e6, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/6694a53bc4894f2ab5af17349abce7ca, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/01d93c30e5ef42dcaf35190f098b28a5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/f3f544e709e3465ebaf2fcaaf6ce7156] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=52.6 K 2024-11-17T21:26:23,034 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37b18980a1ab4233869e9695808d06e6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731878778639 2024-11-17T21:26:23,034 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d48f324a8c4e415b9253f4afb9de8f58, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731878778639 2024-11-17T21:26:23,034 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6694a53bc4894f2ab5af17349abce7ca, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731878778861 2024-11-17T21:26:23,034 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 121a4eefd8c641bbba9a31f4ba8c7363, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731878778861 2024-11-17T21:26:23,034 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting e7b7ad3d46de49e987acf8305f08c3bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731878779247 2024-11-17T21:26:23,034 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01d93c30e5ef42dcaf35190f098b28a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731878779247 2024-11-17T21:26:23,034 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3f544e709e3465ebaf2fcaaf6ce7156, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731878781456 2024-11-17T21:26:23,034 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 0258aba18a314b749c6b36e288f6b31f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731878781456 2024-11-17T21:26:23,046 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#B#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:23,047 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#A#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:23,047 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/b7928bd6175a4c53b21ce08d6e97fd7e is 50, key is test_row_0/B:col10/1731878781456/Put/seqid=0 2024-11-17T21:26:23,047 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/9f2fdf7a8ca14dcfbd1c7b3d57b336f9 is 50, key is test_row_0/A:col10/1731878781456/Put/seqid=0 2024-11-17T21:26:23,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742324_1500 (size=12731) 2024-11-17T21:26:23,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742325_1501 (size=12731) 2024-11-17T21:26:23,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T21:26:23,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:23,108 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-17T21:26:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/c33658160c384d08afc5cf7cdb59f542 is 50, key is test_row_0/A:col10/1731878782178/Put/seqid=0 2024-11-17T21:26:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T21:26:23,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742326_1502 (size=12151) 2024-11-17T21:26:23,126 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/c33658160c384d08afc5cf7cdb59f542 2024-11-17T21:26:23,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/fe0ca493eaf74f4e85573412f9934429 is 50, key is test_row_0/B:col10/1731878782178/Put/seqid=0 2024-11-17T21:26:23,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742327_1503 (size=12151) 2024-11-17T21:26:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:23,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:23,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878843363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:23,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878843364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:23,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878843370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878843371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:23,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878843376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:23,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T21:26:23,464 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/b7928bd6175a4c53b21ce08d6e97fd7e as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b7928bd6175a4c53b21ce08d6e97fd7e 2024-11-17T21:26:23,464 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/9f2fdf7a8ca14dcfbd1c7b3d57b336f9 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/9f2fdf7a8ca14dcfbd1c7b3d57b336f9 2024-11-17T21:26:23,468 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/B of 1e1006bdbe13128ffffb3f217a9e33ec into b7928bd6175a4c53b21ce08d6e97fd7e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:23,468 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/A of 1e1006bdbe13128ffffb3f217a9e33ec into 9f2fdf7a8ca14dcfbd1c7b3d57b336f9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:23,468 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:23,468 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:23,468 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/A, priority=12, startTime=1731878783030; duration=0sec 2024-11-17T21:26:23,468 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/B, priority=12, startTime=1731878783031; duration=0sec 2024-11-17T21:26:23,468 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:23,468 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:23,468 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:A 2024-11-17T21:26:23,468 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:B 2024-11-17T21:26:23,468 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:26:23,469 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:26:23,469 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/C is initiating minor compaction (all files) 2024-11-17T21:26:23,469 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/C in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:23,469 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/54732f2c8d8f40e79aba7443dd6c27e1, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/3359c9b1b6d84b58803024a47a72b895, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/c856148a5d6e4524abb63ac15fa55d29, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/425d999521ac4b38aed2f1d0412884fc] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=47.9 K 2024-11-17T21:26:23,469 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54732f2c8d8f40e79aba7443dd6c27e1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731878778639 2024-11-17T21:26:23,470 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3359c9b1b6d84b58803024a47a72b895, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731878778861 2024-11-17T21:26:23,470 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting c856148a5d6e4524abb63ac15fa55d29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731878779247 2024-11-17T21:26:23,470 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 425d999521ac4b38aed2f1d0412884fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731878781456 2024-11-17T21:26:23,481 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#C#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:23,481 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/3a058ce5ef66435d84ef6a82802131b5 is 50, key is test_row_0/C:col10/1731878781456/Put/seqid=0 2024-11-17T21:26:23,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742328_1504 (size=12731) 2024-11-17T21:26:23,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878843480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878843480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:23,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878843491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878843491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:23,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878843491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:23,664 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/fe0ca493eaf74f4e85573412f9934429 2024-11-17T21:26:23,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/ef1b43c56246419ebf9ba68487cee09d is 50, key is test_row_0/C:col10/1731878782178/Put/seqid=0 2024-11-17T21:26:23,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742329_1505 (size=12151) 2024-11-17T21:26:23,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878843690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:23,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878843690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:23,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878843701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:23,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878843701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878843701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:23,888 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/3a058ce5ef66435d84ef6a82802131b5 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/3a058ce5ef66435d84ef6a82802131b5 2024-11-17T21:26:23,892 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/C of 1e1006bdbe13128ffffb3f217a9e33ec into 3a058ce5ef66435d84ef6a82802131b5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:23,892 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:23,892 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/C, priority=12, startTime=1731878783031; duration=0sec 2024-11-17T21:26:23,892 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:23,892 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:C 2024-11-17T21:26:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T21:26:24,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878844001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878844001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:24,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878844011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:24,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878844011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:24,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878844011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:24,073 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/ef1b43c56246419ebf9ba68487cee09d 2024-11-17T21:26:24,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/c33658160c384d08afc5cf7cdb59f542 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/c33658160c384d08afc5cf7cdb59f542 2024-11-17T21:26:24,080 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/c33658160c384d08afc5cf7cdb59f542, entries=150, sequenceid=253, filesize=11.9 K 2024-11-17T21:26:24,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/fe0ca493eaf74f4e85573412f9934429 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/fe0ca493eaf74f4e85573412f9934429 2024-11-17T21:26:24,085 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/fe0ca493eaf74f4e85573412f9934429, entries=150, sequenceid=253, filesize=11.9 K 2024-11-17T21:26:24,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/ef1b43c56246419ebf9ba68487cee09d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/ef1b43c56246419ebf9ba68487cee09d 2024-11-17T21:26:24,090 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/ef1b43c56246419ebf9ba68487cee09d, entries=150, sequenceid=253, filesize=11.9 K 2024-11-17T21:26:24,091 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 1e1006bdbe13128ffffb3f217a9e33ec in 983ms, sequenceid=253, compaction requested=false 2024-11-17T21:26:24,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:24,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:24,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-17T21:26:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-17T21:26:24,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-17T21:26:24,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2880 sec 2024-11-17T21:26:24,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.2920 sec 2024-11-17T21:26:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:24,527 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:24,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-17T21:26:24,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/84c79f8d9a904b7ca021f5a548c6400d is 50, key is test_row_0/A:col10/1731878784526/Put/seqid=0 2024-11-17T21:26:24,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742330_1506 (size=14741) 2024-11-17T21:26:24,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878844531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878844531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:24,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878844539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:24,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:24,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878844539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:24,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:24,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878844541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:24,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878844651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:24,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878844651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:24,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878844661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:24,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878844861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:24,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878844861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:24,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:24,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878844871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:24,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:24,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:24,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/84c79f8d9a904b7ca021f5a548c6400d 2024-11-17T21:26:24,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/2915ec27d5034995b9b96bac31afa15a is 50, key is test_row_0/B:col10/1731878784526/Put/seqid=0 2024-11-17T21:26:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T21:26:24,953 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:24,953 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-17T21:26:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T21:26:24,955 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:24,956 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:24,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:24,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742331_1507 (size=12301) 2024-11-17T21:26:24,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/2915ec27d5034995b9b96bac31afa15a 2024-11-17T21:26:24,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/275906c9d1f7405c9673bf5c47d1b612 is 50, key is test_row_0/C:col10/1731878784526/Put/seqid=0 2024-11-17T21:26:25,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742332_1508 (size=12301) 2024-11-17T21:26:25,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/275906c9d1f7405c9673bf5c47d1b612 2024-11-17T21:26:25,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/84c79f8d9a904b7ca021f5a548c6400d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/84c79f8d9a904b7ca021f5a548c6400d 2024-11-17T21:26:25,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/84c79f8d9a904b7ca021f5a548c6400d, entries=200, sequenceid=285, filesize=14.4 K 2024-11-17T21:26:25,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/2915ec27d5034995b9b96bac31afa15a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/2915ec27d5034995b9b96bac31afa15a 2024-11-17T21:26:25,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/2915ec27d5034995b9b96bac31afa15a, entries=150, sequenceid=285, filesize=12.0 K 2024-11-17T21:26:25,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/275906c9d1f7405c9673bf5c47d1b612 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/275906c9d1f7405c9673bf5c47d1b612 2024-11-17T21:26:25,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/275906c9d1f7405c9673bf5c47d1b612, entries=150, sequenceid=285, filesize=12.0 K 2024-11-17T21:26:25,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 1e1006bdbe13128ffffb3f217a9e33ec in 517ms, sequenceid=285, compaction requested=true 2024-11-17T21:26:25,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:25,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:26:25,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:25,044 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:25,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:26:25,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:25,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:26:25,044 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:25,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:25,045 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:25,045 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:25,045 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/A is initiating minor compaction (all files) 2024-11-17T21:26:25,045 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/B is initiating minor compaction (all files) 2024-11-17T21:26:25,045 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/A in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:25,045 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/B in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:25,045 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b7928bd6175a4c53b21ce08d6e97fd7e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/fe0ca493eaf74f4e85573412f9934429, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/2915ec27d5034995b9b96bac31afa15a] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=36.3 K 2024-11-17T21:26:25,045 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/9f2fdf7a8ca14dcfbd1c7b3d57b336f9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/c33658160c384d08afc5cf7cdb59f542, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/84c79f8d9a904b7ca021f5a548c6400d] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=38.7 K 2024-11-17T21:26:25,045 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7928bd6175a4c53b21ce08d6e97fd7e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731878781456 2024-11-17T21:26:25,045 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe0ca493eaf74f4e85573412f9934429, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731878782177 2024-11-17T21:26:25,047 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f2fdf7a8ca14dcfbd1c7b3d57b336f9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731878781456 2024-11-17T21:26:25,047 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2915ec27d5034995b9b96bac31afa15a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731878783362 2024-11-17T21:26:25,047 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting c33658160c384d08afc5cf7cdb59f542, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731878782177 2024-11-17T21:26:25,048 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 84c79f8d9a904b7ca021f5a548c6400d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731878783362 2024-11-17T21:26:25,053 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#B#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:25,054 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/4051767743a7411ebeeff6d4c5cce1f4 is 50, key is test_row_0/B:col10/1731878784526/Put/seqid=0 2024-11-17T21:26:25,059 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#A#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:25,060 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/87c1102bd3d24bbd9c7f9544acad1429 is 50, key is test_row_0/A:col10/1731878784526/Put/seqid=0 2024-11-17T21:26:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T21:26:25,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742334_1510 (size=12983) 2024-11-17T21:26:25,086 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/87c1102bd3d24bbd9c7f9544acad1429 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/87c1102bd3d24bbd9c7f9544acad1429 2024-11-17T21:26:25,090 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/A of 1e1006bdbe13128ffffb3f217a9e33ec into 87c1102bd3d24bbd9c7f9544acad1429(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:25,090 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:25,090 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/A, priority=13, startTime=1731878785044; duration=0sec 2024-11-17T21:26:25,090 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:25,090 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:A 2024-11-17T21:26:25,090 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:25,091 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:25,091 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/C is initiating minor compaction (all files) 2024-11-17T21:26:25,091 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/C in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:25,091 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/3a058ce5ef66435d84ef6a82802131b5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/ef1b43c56246419ebf9ba68487cee09d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/275906c9d1f7405c9673bf5c47d1b612] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=36.3 K 2024-11-17T21:26:25,091 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a058ce5ef66435d84ef6a82802131b5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731878781456 2024-11-17T21:26:25,091 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting ef1b43c56246419ebf9ba68487cee09d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731878782177 2024-11-17T21:26:25,091 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 275906c9d1f7405c9673bf5c47d1b612, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731878783362 2024-11-17T21:26:25,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742333_1509 (size=12983) 2024-11-17T21:26:25,097 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#C#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:25,097 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/43eee8b7965e44fdb1f3ba1ece09966b is 50, key is test_row_0/C:col10/1731878784526/Put/seqid=0 2024-11-17T21:26:25,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T21:26:25,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:25,108 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-17T21:26:25,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742335_1511 (size=12983) 2024-11-17T21:26:25,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/1d36780ab3c144be947f5969b39f3586 is 50, key is test_row_0/A:col10/1731878784537/Put/seqid=0 2024-11-17T21:26:25,117 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/4051767743a7411ebeeff6d4c5cce1f4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/4051767743a7411ebeeff6d4c5cce1f4 2024-11-17T21:26:25,124 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/43eee8b7965e44fdb1f3ba1ece09966b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/43eee8b7965e44fdb1f3ba1ece09966b 2024-11-17T21:26:25,125 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/B of 1e1006bdbe13128ffffb3f217a9e33ec into 4051767743a7411ebeeff6d4c5cce1f4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:25,125 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:25,125 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/B, priority=13, startTime=1731878785044; duration=0sec 2024-11-17T21:26:25,125 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:25,125 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:B 2024-11-17T21:26:25,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742336_1512 (size=9857) 2024-11-17T21:26:25,136 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/C of 1e1006bdbe13128ffffb3f217a9e33ec into 43eee8b7965e44fdb1f3ba1ece09966b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:25,136 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:25,136 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/C, priority=13, startTime=1731878785044; duration=0sec 2024-11-17T21:26:25,136 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:25,136 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:C 2024-11-17T21:26:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:25,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:25,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878845238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:25,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878845239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:25,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878845240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:25,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T21:26:25,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878845350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878845350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:25,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878845351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:25,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:25,535 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/1d36780ab3c144be947f5969b39f3586 2024-11-17T21:26:25,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/20e876beca104ca0b5b57afd6e50fedc is 50, key is test_row_0/B:col10/1731878784537/Put/seqid=0 2024-11-17T21:26:25,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878845551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:25,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878845551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:25,553 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in 1e1006bdbe13128ffffb3f217a9e33ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,553 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in 1e1006bdbe13128ffffb3f217a9e33ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:25,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878845561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:25,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742337_1513 (size=9857) 2024-11-17T21:26:25,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878845562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:25,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878845562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T21:26:25,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:25,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878845871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:25,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878845871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:25,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878845880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:25,968 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/20e876beca104ca0b5b57afd6e50fedc 2024-11-17T21:26:25,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/beadb7e7e62247cc820e333c1884bdfd is 50, key is test_row_0/C:col10/1731878784537/Put/seqid=0 2024-11-17T21:26:25,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742338_1514 (size=9857) 2024-11-17T21:26:25,980 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/beadb7e7e62247cc820e333c1884bdfd 2024-11-17T21:26:25,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/1d36780ab3c144be947f5969b39f3586 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/1d36780ab3c144be947f5969b39f3586 2024-11-17T21:26:25,991 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/1d36780ab3c144be947f5969b39f3586, entries=100, sequenceid=293, filesize=9.6 K 2024-11-17T21:26:25,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/20e876beca104ca0b5b57afd6e50fedc as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/20e876beca104ca0b5b57afd6e50fedc 2024-11-17T21:26:25,994 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/20e876beca104ca0b5b57afd6e50fedc, entries=100, sequenceid=293, filesize=9.6 K 2024-11-17T21:26:25,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/beadb7e7e62247cc820e333c1884bdfd as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/beadb7e7e62247cc820e333c1884bdfd 2024-11-17T21:26:25,998 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/beadb7e7e62247cc820e333c1884bdfd, entries=100, sequenceid=293, filesize=9.6 K 2024-11-17T21:26:26,000 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 1e1006bdbe13128ffffb3f217a9e33ec in 892ms, sequenceid=293, compaction requested=false 2024-11-17T21:26:26,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:26,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:26,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-17T21:26:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-17T21:26:26,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-17T21:26:26,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0450 sec 2024-11-17T21:26:26,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.0520 sec 2024-11-17T21:26:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T21:26:26,092 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:26,093 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-17T21:26:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T21:26:26,095 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:26,095 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:26,095 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T21:26:26,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T21:26:26,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:26,247 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-17T21:26:26,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/7a25fed177944071900233c3c2d97cd8 is 50, key is test_row_0/A:col10/1731878785238/Put/seqid=0 2024-11-17T21:26:26,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742339_1515 (size=12301) 2024-11-17T21:26:26,277 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/7a25fed177944071900233c3c2d97cd8 2024-11-17T21:26:26,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/a893a5d39aa7453bb58ee2ed6e402e68 is 50, key is test_row_0/B:col10/1731878785238/Put/seqid=0 2024-11-17T21:26:26,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742340_1516 (size=12301) 2024-11-17T21:26:26,308 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/a893a5d39aa7453bb58ee2ed6e402e68 2024-11-17T21:26:26,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/89e2af497ede47999a757566a46cb4fb is 50, key is test_row_0/C:col10/1731878785238/Put/seqid=0 2024-11-17T21:26:26,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742341_1517 (size=12301) 2024-11-17T21:26:26,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:26,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878846401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:26,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878846401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:26,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878846403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T21:26:26,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878846511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:26,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:26,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:26,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:26,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878846522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:26,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:26,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:26,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T21:26:26,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878846721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:26,726 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/89e2af497ede47999a757566a46cb4fb 2024-11-17T21:26:26,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/7a25fed177944071900233c3c2d97cd8 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/7a25fed177944071900233c3c2d97cd8 2024-11-17T21:26:26,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878846731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:26,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:26,736 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/7a25fed177944071900233c3c2d97cd8, entries=150, sequenceid=324, filesize=12.0 K 2024-11-17T21:26:26,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/a893a5d39aa7453bb58ee2ed6e402e68 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/a893a5d39aa7453bb58ee2ed6e402e68 2024-11-17T21:26:26,744 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/a893a5d39aa7453bb58ee2ed6e402e68, entries=150, sequenceid=324, filesize=12.0 K 2024-11-17T21:26:26,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/89e2af497ede47999a757566a46cb4fb as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/89e2af497ede47999a757566a46cb4fb 2024-11-17T21:26:26,750 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/89e2af497ede47999a757566a46cb4fb, entries=150, sequenceid=324, filesize=12.0 K 2024-11-17T21:26:26,751 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 1e1006bdbe13128ffffb3f217a9e33ec in 504ms, sequenceid=324, compaction requested=true 2024-11-17T21:26:26,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:26,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:26,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-17T21:26:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-17T21:26:26,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-17T21:26:26,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 657 msec 2024-11-17T21:26:26,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 661 msec 2024-11-17T21:26:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:27,044 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:27,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:26:27,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/2cce80ecc5b14d8f900cef658265240c is 50, key is test_row_0/A:col10/1731878787043/Put/seqid=0 2024-11-17T21:26:27,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742342_1518 (size=14741) 2024-11-17T21:26:27,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/2cce80ecc5b14d8f900cef658265240c 2024-11-17T21:26:27,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/b0009a8daf9c4373bd27071da27766b7 is 50, key is test_row_0/B:col10/1731878787043/Put/seqid=0 2024-11-17T21:26:27,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742343_1519 (size=12301) 2024-11-17T21:26:27,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878847146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:27,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878847149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T21:26:27,231 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:27,231 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-17T21:26:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T21:26:27,233 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:27,233 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:27,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:27,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878847261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:27,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878847261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T21:26:27,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T21:26:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:27,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:27,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:27,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45826 deadline: 1731878847421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:27,428 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_2' in 1e1006bdbe13128ffffb3f217a9e33ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:27,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:27,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/b0009a8daf9c4373bd27071da27766b7 2024-11-17T21:26:27,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878847471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:27,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:27,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/c43d0d1cfe74481bac89e56c0c2ebf9b is 50, key is test_row_0/C:col10/1731878787043/Put/seqid=0 2024-11-17T21:26:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878847471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:27,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742344_1520 (size=12301) 2024-11-17T21:26:27,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T21:26:27,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:27,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:27,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:27,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:27,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T21:26:27,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45804 deadline: 1731878847571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:27,576 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in 1e1006bdbe13128ffffb3f217a9e33ec of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:27,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:27,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731878847581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:27,585 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in 1e1006bdbe13128ffffb3f217a9e33ec of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:27,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T21:26:27,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:27,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:27,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:27,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:27,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:27,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45790 deadline: 1731878847781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:27,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45824 deadline: 1731878847781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., hostname=a313eea8709e,46445,1731878642828, seqNum=2 because the exception is null or not the one we care about 2024-11-17T21:26:27,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T21:26:27,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:27,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:27,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:27,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:27,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T21:26:27,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/c43d0d1cfe74481bac89e56c0c2ebf9b 2024-11-17T21:26:27,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/2cce80ecc5b14d8f900cef658265240c as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/2cce80ecc5b14d8f900cef658265240c 2024-11-17T21:26:27,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/2cce80ecc5b14d8f900cef658265240c, entries=200, sequenceid=335, filesize=14.4 K 2024-11-17T21:26:27,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/b0009a8daf9c4373bd27071da27766b7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b0009a8daf9c4373bd27071da27766b7 2024-11-17T21:26:27,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b0009a8daf9c4373bd27071da27766b7, entries=150, sequenceid=335, filesize=12.0 K 2024-11-17T21:26:27,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/c43d0d1cfe74481bac89e56c0c2ebf9b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/c43d0d1cfe74481bac89e56c0c2ebf9b 2024-11-17T21:26:27,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/c43d0d1cfe74481bac89e56c0c2ebf9b, entries=150, sequenceid=335, filesize=12.0 K 2024-11-17T21:26:27,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1e1006bdbe13128ffffb3f217a9e33ec in 871ms, sequenceid=335, compaction requested=true 2024-11-17T21:26:27,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:27,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:26:27,916 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:26:27,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:27,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:26:27,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:27,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1006bdbe13128ffffb3f217a9e33ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:26:27,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T21:26:27,916 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:26:27,917 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49882 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:26:27,917 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:26:27,917 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/B is initiating minor compaction (all files) 2024-11-17T21:26:27,917 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/A is initiating minor compaction (all files) 2024-11-17T21:26:27,917 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/A in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:27,917 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/B in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:27,917 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/4051767743a7411ebeeff6d4c5cce1f4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/20e876beca104ca0b5b57afd6e50fedc, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/a893a5d39aa7453bb58ee2ed6e402e68, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b0009a8daf9c4373bd27071da27766b7] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=46.3 K 2024-11-17T21:26:27,917 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/87c1102bd3d24bbd9c7f9544acad1429, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/1d36780ab3c144be947f5969b39f3586, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/7a25fed177944071900233c3c2d97cd8, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/2cce80ecc5b14d8f900cef658265240c] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=48.7 K 2024-11-17T21:26:27,917 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87c1102bd3d24bbd9c7f9544acad1429, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731878783362 2024-11-17T21:26:27,918 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 4051767743a7411ebeeff6d4c5cce1f4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731878783362 2024-11-17T21:26:27,918 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d36780ab3c144be947f5969b39f3586, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731878784537 2024-11-17T21:26:27,918 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 20e876beca104ca0b5b57afd6e50fedc, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731878784537 2024-11-17T21:26:27,918 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a25fed177944071900233c3c2d97cd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731878785237 2024-11-17T21:26:27,918 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting a893a5d39aa7453bb58ee2ed6e402e68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731878785237 2024-11-17T21:26:27,918 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cce80ecc5b14d8f900cef658265240c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731878786382 2024-11-17T21:26:27,919 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting b0009a8daf9c4373bd27071da27766b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731878787033 2024-11-17T21:26:27,931 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#A#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:27,932 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/f176deccbb124c50a8a34886dec2bb57 is 50, key is test_row_0/A:col10/1731878787043/Put/seqid=0 2024-11-17T21:26:27,935 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:27,935 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/bb56c77fbb6f4c009248e0b4b79b2cd7 is 50, key is test_row_0/B:col10/1731878787043/Put/seqid=0 2024-11-17T21:26:27,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742345_1521 (size=13119) 2024-11-17T21:26:27,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742346_1522 (size=13119) 2024-11-17T21:26:27,986 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/f176deccbb124c50a8a34886dec2bb57 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/f176deccbb124c50a8a34886dec2bb57 2024-11-17T21:26:27,995 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/A of 1e1006bdbe13128ffffb3f217a9e33ec into f176deccbb124c50a8a34886dec2bb57(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:27,995 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:27,995 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/A, priority=12, startTime=1731878787915; duration=0sec 2024-11-17T21:26:27,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T21:26:27,995 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:27,995 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:A 2024-11-17T21:26:27,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:27,996 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:26:27,996 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T21:26:27,998 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:26:27,998 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): 1e1006bdbe13128ffffb3f217a9e33ec/C is initiating minor compaction (all files) 2024-11-17T21:26:27,998 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1006bdbe13128ffffb3f217a9e33ec/C in TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:27,998 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/43eee8b7965e44fdb1f3ba1ece09966b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/beadb7e7e62247cc820e333c1884bdfd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/89e2af497ede47999a757566a46cb4fb, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/c43d0d1cfe74481bac89e56c0c2ebf9b] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp, totalSize=46.3 K 2024-11-17T21:26:27,999 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43eee8b7965e44fdb1f3ba1ece09966b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731878783362 2024-11-17T21:26:27,999 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting beadb7e7e62247cc820e333c1884bdfd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731878784537 2024-11-17T21:26:28,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/a630fe2ea94141f7846009f2bb5c06e5 is 50, key is test_row_0/A:col10/1731878787148/Put/seqid=0 2024-11-17T21:26:28,002 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89e2af497ede47999a757566a46cb4fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731878785237 2024-11-17T21:26:28,002 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting c43d0d1cfe74481bac89e56c0c2ebf9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731878787033 2024-11-17T21:26:28,008 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1006bdbe13128ffffb3f217a9e33ec#C#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:28,009 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/52eee623d7c64f6ca16e24ef985d0039 is 50, key is test_row_0/C:col10/1731878787043/Put/seqid=0 2024-11-17T21:26:28,011 INFO [Thread-2028 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2028. 2024-11-17T21:26:28,011 DEBUG [Thread-2028 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:28,011 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:28,011 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:28,011 INFO [Thread-2022 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2022. 2024-11-17T21:26:28,011 DEBUG [Thread-2022 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:28,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:28,011 DEBUG [Thread-2022 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:28,011 DEBUG [Thread-2022 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:28,012 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:28,018 INFO [Thread-2020 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2020. 2024-11-17T21:26:28,018 DEBUG [Thread-2020 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:28,018 DEBUG [Thread-2020 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:28,018 DEBUG [Thread-2020 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:28,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:28,019 INFO [Thread-2024 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2024. 2024-11-17T21:26:28,020 DEBUG [Thread-2024 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:28,020 DEBUG [Thread-2024 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:28,020 DEBUG [Thread-2024 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:28,020 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:28,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742348_1524 (size=12301) 2024-11-17T21:26:28,021 INFO [Thread-2026 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2026. 2024-11-17T21:26:28,021 DEBUG [Thread-2026 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:28,021 DEBUG [Thread-2026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:28,021 DEBUG [Thread-2026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:28,022 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:28,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742347_1523 (size=13119) 2024-11-17T21:26:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:28,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. as already flushing 2024-11-17T21:26:28,294 INFO [Thread-2015 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2015. 2024-11-17T21:26:28,294 DEBUG [Thread-2015 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:28,294 DEBUG [Thread-2015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:28,294 DEBUG [Thread-2015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:28,294 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:28,295 INFO [Thread-2013 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2013. 2024-11-17T21:26:28,295 DEBUG [Thread-2013 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:28,295 DEBUG [Thread-2013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:28,295 DEBUG [Thread-2013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:28,295 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T21:26:28,396 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/bb56c77fbb6f4c009248e0b4b79b2cd7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/bb56c77fbb6f4c009248e0b4b79b2cd7 2024-11-17T21:26:28,401 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/B of 1e1006bdbe13128ffffb3f217a9e33ec into bb56c77fbb6f4c009248e0b4b79b2cd7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:28,401 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:28,401 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/B, priority=12, startTime=1731878787916; duration=0sec 2024-11-17T21:26:28,401 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:28,401 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:B 2024-11-17T21:26:28,421 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/a630fe2ea94141f7846009f2bb5c06e5 2024-11-17T21:26:28,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/f34d547f6df94d58958d49cba7b8b9a9 is 50, key is test_row_0/B:col10/1731878787148/Put/seqid=0 2024-11-17T21:26:28,433 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/52eee623d7c64f6ca16e24ef985d0039 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/52eee623d7c64f6ca16e24ef985d0039 2024-11-17T21:26:28,438 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1006bdbe13128ffffb3f217a9e33ec/C of 1e1006bdbe13128ffffb3f217a9e33ec into 52eee623d7c64f6ca16e24ef985d0039(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:28,438 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:28,438 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., storeName=1e1006bdbe13128ffffb3f217a9e33ec/C, priority=12, startTime=1731878787916; duration=0sec 2024-11-17T21:26:28,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742349_1525 (size=12301) 2024-11-17T21:26:28,438 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:28,438 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1006bdbe13128ffffb3f217a9e33ec:C 2024-11-17T21:26:28,439 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/f34d547f6df94d58958d49cba7b8b9a9 2024-11-17T21:26:28,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/e36fff532e9b419abfded0ca3eedad2f is 50, key is test_row_0/C:col10/1731878787148/Put/seqid=0 2024-11-17T21:26:28,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742350_1526 (size=12301) 2024-11-17T21:26:28,451 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/e36fff532e9b419abfded0ca3eedad2f 2024-11-17T21:26:28,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/a630fe2ea94141f7846009f2bb5c06e5 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/a630fe2ea94141f7846009f2bb5c06e5 2024-11-17T21:26:28,463 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/a630fe2ea94141f7846009f2bb5c06e5, entries=150, sequenceid=361, filesize=12.0 K 2024-11-17T21:26:28,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/f34d547f6df94d58958d49cba7b8b9a9 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/f34d547f6df94d58958d49cba7b8b9a9 2024-11-17T21:26:28,468 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/f34d547f6df94d58958d49cba7b8b9a9, entries=150, sequenceid=361, filesize=12.0 K 2024-11-17T21:26:28,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/e36fff532e9b419abfded0ca3eedad2f as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/e36fff532e9b419abfded0ca3eedad2f 2024-11-17T21:26:28,476 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/e36fff532e9b419abfded0ca3eedad2f, entries=150, sequenceid=361, filesize=12.0 K 2024-11-17T21:26:28,477 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=13.42 KB/13740 for 1e1006bdbe13128ffffb3f217a9e33ec in 481ms, sequenceid=361, compaction requested=false 2024-11-17T21:26:28,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 1e1006bdbe13128ffffb3f217a9e33ec: 2024-11-17T21:26:28,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:28,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-17T21:26:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-17T21:26:28,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-17T21:26:28,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2460 sec 2024-11-17T21:26:28,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.2500 sec 2024-11-17T21:26:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T21:26:29,381 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:29,453 INFO [Thread-2009 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2009. 2024-11-17T21:26:29,453 DEBUG [Thread-2009 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:29,453 DEBUG [Thread-2009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:29,453 DEBUG [Thread-2009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:29,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:31,090 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T21:26:31,593 INFO [Thread-2011 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2011. 2024-11-17T21:26:31,593 DEBUG [Thread-2011 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:31,593 DEBUG [Thread-2011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:31,593 DEBUG [Thread-2011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:31,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:31,612 INFO [Thread-2017 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2017. 2024-11-17T21:26:31,612 DEBUG [Thread-2017 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:31,612 DEBUG [Thread-2017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:31,612 DEBUG [Thread-2017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:31,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1578 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4730 rows 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1570 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4707 rows 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1572 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4710 rows 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1571 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4713 rows 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1570 2024-11-17T21:26:31,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4709 rows 2024-11-17T21:26:31,613 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T21:26:31,613 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T21:26:31,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:31,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:31,613 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:31,614 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T21:26:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T21:26:31,616 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878791616"}]},"ts":"1731878791616"} 2024-11-17T21:26:31,617 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T21:26:31,617 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T21:26:31,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T21:26:31,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1006bdbe13128ffffb3f217a9e33ec, UNASSIGN}] 2024-11-17T21:26:31,619 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1006bdbe13128ffffb3f217a9e33ec, UNASSIGN 2024-11-17T21:26:31,620 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=1e1006bdbe13128ffffb3f217a9e33ec, regionState=CLOSING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:26:31,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1006bdbe13128ffffb3f217a9e33ec, UNASSIGN because future has completed 2024-11-17T21:26:31,621 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T21:26:31,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:26:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T21:26:31,774 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:31,774 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T21:26:31,774 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 1e1006bdbe13128ffffb3f217a9e33ec, disabling compactions & flushes 2024-11-17T21:26:31,774 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:31,774 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:31,774 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. after waiting 0 ms 2024-11-17T21:26:31,774 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:31,774 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 1e1006bdbe13128ffffb3f217a9e33ec 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-17T21:26:31,777 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/811e6b66c7844d1db03ef0fb8c2822b4 is 50, key is test_row_1/A:col10/1731878791611/Put/seqid=0 2024-11-17T21:26:31,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742351_1527 (size=9857) 2024-11-17T21:26:31,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T21:26:32,181 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/811e6b66c7844d1db03ef0fb8c2822b4 2024-11-17T21:26:32,186 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/d99d02d9f9404988bd519c65f655a60e is 50, key is test_row_1/B:col10/1731878791611/Put/seqid=0 2024-11-17T21:26:32,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742352_1528 (size=9857) 2024-11-17T21:26:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T21:26:32,590 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/d99d02d9f9404988bd519c65f655a60e 2024-11-17T21:26:32,595 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/1278839bc9534f258e71c1a7915f3bad is 50, key is test_row_1/C:col10/1731878791611/Put/seqid=0 2024-11-17T21:26:32,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742353_1529 (size=9857) 2024-11-17T21:26:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T21:26:33,000 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/1278839bc9534f258e71c1a7915f3bad 2024-11-17T21:26:33,004 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/A/811e6b66c7844d1db03ef0fb8c2822b4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/811e6b66c7844d1db03ef0fb8c2822b4 2024-11-17T21:26:33,007 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/811e6b66c7844d1db03ef0fb8c2822b4, entries=100, sequenceid=371, filesize=9.6 K 2024-11-17T21:26:33,007 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/B/d99d02d9f9404988bd519c65f655a60e as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/d99d02d9f9404988bd519c65f655a60e 2024-11-17T21:26:33,010 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/d99d02d9f9404988bd519c65f655a60e, entries=100, sequenceid=371, filesize=9.6 K 2024-11-17T21:26:33,011 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/.tmp/C/1278839bc9534f258e71c1a7915f3bad as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/1278839bc9534f258e71c1a7915f3bad 2024-11-17T21:26:33,013 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/1278839bc9534f258e71c1a7915f3bad, entries=100, sequenceid=371, filesize=9.6 K 2024-11-17T21:26:33,014 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1e1006bdbe13128ffffb3f217a9e33ec in 1240ms, sequenceid=371, compaction requested=true 2024-11-17T21:26:33,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/17f927a3c2b444fa91e7a4307ddc97f7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/5a97dce264054687a2cad48e79d82fd9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/6fbb1b73307745c49593118197adedab, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/b870a63e9d0f4783b6f665fe3bdbe625, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/43355e3a9ac04524bc964c5266df74a4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/31e24a8e816845fcbf9eb20b2783186c, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/319a6f71e70749b3be70fa68d743b74e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/72060dca48d94133a1d043f82415e1b4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/cadf4cb042be4695a30353c72a7b466e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/e09a3b195d2b4b55a80adcfde3674c0a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/d4ee5e50444c45a3a04bd6070cc0ebe4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/d2f69149038a40d9a5f32779a4c62229, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/bdc65db635504ce2b70109b81869e3e0, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/37b18980a1ab4233869e9695808d06e6, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/6694a53bc4894f2ab5af17349abce7ca, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/01d93c30e5ef42dcaf35190f098b28a5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/f3f544e709e3465ebaf2fcaaf6ce7156, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/9f2fdf7a8ca14dcfbd1c7b3d57b336f9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/c33658160c384d08afc5cf7cdb59f542, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/84c79f8d9a904b7ca021f5a548c6400d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/87c1102bd3d24bbd9c7f9544acad1429, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/1d36780ab3c144be947f5969b39f3586, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/7a25fed177944071900233c3c2d97cd8, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/2cce80ecc5b14d8f900cef658265240c] to archive 2024-11-17T21:26:33,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:26:33,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/17f927a3c2b444fa91e7a4307ddc97f7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/17f927a3c2b444fa91e7a4307ddc97f7 2024-11-17T21:26:33,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/5a97dce264054687a2cad48e79d82fd9 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/5a97dce264054687a2cad48e79d82fd9 2024-11-17T21:26:33,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/6fbb1b73307745c49593118197adedab to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/6fbb1b73307745c49593118197adedab 2024-11-17T21:26:33,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/b870a63e9d0f4783b6f665fe3bdbe625 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/b870a63e9d0f4783b6f665fe3bdbe625 2024-11-17T21:26:33,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/43355e3a9ac04524bc964c5266df74a4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/43355e3a9ac04524bc964c5266df74a4 2024-11-17T21:26:33,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/31e24a8e816845fcbf9eb20b2783186c to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/31e24a8e816845fcbf9eb20b2783186c 2024-11-17T21:26:33,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/319a6f71e70749b3be70fa68d743b74e to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/319a6f71e70749b3be70fa68d743b74e 2024-11-17T21:26:33,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/72060dca48d94133a1d043f82415e1b4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/72060dca48d94133a1d043f82415e1b4 2024-11-17T21:26:33,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/cadf4cb042be4695a30353c72a7b466e to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/cadf4cb042be4695a30353c72a7b466e 2024-11-17T21:26:33,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/e09a3b195d2b4b55a80adcfde3674c0a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/e09a3b195d2b4b55a80adcfde3674c0a 2024-11-17T21:26:33,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/d4ee5e50444c45a3a04bd6070cc0ebe4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/d4ee5e50444c45a3a04bd6070cc0ebe4 2024-11-17T21:26:33,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/d2f69149038a40d9a5f32779a4c62229 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/d2f69149038a40d9a5f32779a4c62229 2024-11-17T21:26:33,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/bdc65db635504ce2b70109b81869e3e0 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/bdc65db635504ce2b70109b81869e3e0 2024-11-17T21:26:33,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/37b18980a1ab4233869e9695808d06e6 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/37b18980a1ab4233869e9695808d06e6 2024-11-17T21:26:33,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/6694a53bc4894f2ab5af17349abce7ca to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/6694a53bc4894f2ab5af17349abce7ca 2024-11-17T21:26:33,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/01d93c30e5ef42dcaf35190f098b28a5 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/01d93c30e5ef42dcaf35190f098b28a5 2024-11-17T21:26:33,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/f3f544e709e3465ebaf2fcaaf6ce7156 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/f3f544e709e3465ebaf2fcaaf6ce7156 2024-11-17T21:26:33,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/9f2fdf7a8ca14dcfbd1c7b3d57b336f9 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/9f2fdf7a8ca14dcfbd1c7b3d57b336f9 2024-11-17T21:26:33,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/c33658160c384d08afc5cf7cdb59f542 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/c33658160c384d08afc5cf7cdb59f542 2024-11-17T21:26:33,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/84c79f8d9a904b7ca021f5a548c6400d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/84c79f8d9a904b7ca021f5a548c6400d 2024-11-17T21:26:33,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/87c1102bd3d24bbd9c7f9544acad1429 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/87c1102bd3d24bbd9c7f9544acad1429 2024-11-17T21:26:33,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/1d36780ab3c144be947f5969b39f3586 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/1d36780ab3c144be947f5969b39f3586 2024-11-17T21:26:33,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/7a25fed177944071900233c3c2d97cd8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/7a25fed177944071900233c3c2d97cd8 2024-11-17T21:26:33,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/2cce80ecc5b14d8f900cef658265240c to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/2cce80ecc5b14d8f900cef658265240c 2024-11-17T21:26:33,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/9f384543906445c3a4dae5b762831b40, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/ca2bf1f3ca2941969f60ac93480ca669, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/0e3a418f26174608bdaf3e0e205f6557, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/f4dbcb1ad32a4fbebfec7a8cba48414b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b228388d49e344eea2f76d4aec737f92, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/8a34e3e8762044b09dc4f7895a2b49e6, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/5e88586c16d5412795c12ef95323d10b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/cce222e7613f48b997d7c34ab8bae5c6, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/6d9fe119c0c94d78bbc367bafcb417ca, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/9213b5209a5f4ab59e67a9b33b1a52ce, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/60c30f129a414912ac0bd820534c6fbf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/26e954e4cb3a41308ba7102bec0640ae, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/d48f324a8c4e415b9253f4afb9de8f58, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/3d65950e553d4fa3a9eb7c69d6e3e64a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/121a4eefd8c641bbba9a31f4ba8c7363, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/e7b7ad3d46de49e987acf8305f08c3bd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b7928bd6175a4c53b21ce08d6e97fd7e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/0258aba18a314b749c6b36e288f6b31f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/fe0ca493eaf74f4e85573412f9934429, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/4051767743a7411ebeeff6d4c5cce1f4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/2915ec27d5034995b9b96bac31afa15a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/20e876beca104ca0b5b57afd6e50fedc, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/a893a5d39aa7453bb58ee2ed6e402e68, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b0009a8daf9c4373bd27071da27766b7] to archive 2024-11-17T21:26:33,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:26:33,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/9f384543906445c3a4dae5b762831b40 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/9f384543906445c3a4dae5b762831b40 2024-11-17T21:26:33,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/ca2bf1f3ca2941969f60ac93480ca669 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/ca2bf1f3ca2941969f60ac93480ca669 2024-11-17T21:26:33,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/0e3a418f26174608bdaf3e0e205f6557 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/0e3a418f26174608bdaf3e0e205f6557 2024-11-17T21:26:33,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/f4dbcb1ad32a4fbebfec7a8cba48414b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/f4dbcb1ad32a4fbebfec7a8cba48414b 2024-11-17T21:26:33,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b228388d49e344eea2f76d4aec737f92 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b228388d49e344eea2f76d4aec737f92 2024-11-17T21:26:33,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/8a34e3e8762044b09dc4f7895a2b49e6 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/8a34e3e8762044b09dc4f7895a2b49e6 2024-11-17T21:26:33,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/5e88586c16d5412795c12ef95323d10b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/5e88586c16d5412795c12ef95323d10b 2024-11-17T21:26:33,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/cce222e7613f48b997d7c34ab8bae5c6 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/cce222e7613f48b997d7c34ab8bae5c6 2024-11-17T21:26:33,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/6d9fe119c0c94d78bbc367bafcb417ca to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/6d9fe119c0c94d78bbc367bafcb417ca 2024-11-17T21:26:33,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/9213b5209a5f4ab59e67a9b33b1a52ce to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/9213b5209a5f4ab59e67a9b33b1a52ce 2024-11-17T21:26:33,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/60c30f129a414912ac0bd820534c6fbf to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/60c30f129a414912ac0bd820534c6fbf 2024-11-17T21:26:33,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/26e954e4cb3a41308ba7102bec0640ae to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/26e954e4cb3a41308ba7102bec0640ae 2024-11-17T21:26:33,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/d48f324a8c4e415b9253f4afb9de8f58 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/d48f324a8c4e415b9253f4afb9de8f58 2024-11-17T21:26:33,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/3d65950e553d4fa3a9eb7c69d6e3e64a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/3d65950e553d4fa3a9eb7c69d6e3e64a 2024-11-17T21:26:33,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/121a4eefd8c641bbba9a31f4ba8c7363 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/121a4eefd8c641bbba9a31f4ba8c7363 2024-11-17T21:26:33,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/e7b7ad3d46de49e987acf8305f08c3bd to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/e7b7ad3d46de49e987acf8305f08c3bd 2024-11-17T21:26:33,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b7928bd6175a4c53b21ce08d6e97fd7e to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b7928bd6175a4c53b21ce08d6e97fd7e 2024-11-17T21:26:33,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/0258aba18a314b749c6b36e288f6b31f to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/0258aba18a314b749c6b36e288f6b31f 2024-11-17T21:26:33,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/fe0ca493eaf74f4e85573412f9934429 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/fe0ca493eaf74f4e85573412f9934429 2024-11-17T21:26:33,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/4051767743a7411ebeeff6d4c5cce1f4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/4051767743a7411ebeeff6d4c5cce1f4 2024-11-17T21:26:33,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/2915ec27d5034995b9b96bac31afa15a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/2915ec27d5034995b9b96bac31afa15a 2024-11-17T21:26:33,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/20e876beca104ca0b5b57afd6e50fedc to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/20e876beca104ca0b5b57afd6e50fedc 2024-11-17T21:26:33,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/a893a5d39aa7453bb58ee2ed6e402e68 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/a893a5d39aa7453bb58ee2ed6e402e68 2024-11-17T21:26:33,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b0009a8daf9c4373bd27071da27766b7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/b0009a8daf9c4373bd27071da27766b7 2024-11-17T21:26:33,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/40ed739b481f4795ad5ca352602d79f7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/2bdb7f5eff1c48099543da4dbcecca46, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/bf48abf8380a4478953a49ef232d9113, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/a3b01a3359074b159b2261dfb4beabda, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/b58db119438a4582bd59a2578ad41e27, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/254464c7ef7c4b14b62cea08b91e0be6, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/d234c8913b344a27a40866318545c523, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/43cdc12ba69c4b11bbe983abb6f94b3a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/39ec304604194e53b7a1b5f07329bd49, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/7068c193f8ae4c99b3ee14c7cc955b28, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/841db4ef4e56493f979720ac3688f2c8, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/4a76ae085aec4aad91de82ce72e476f4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/54732f2c8d8f40e79aba7443dd6c27e1, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/b789807916ea414d9e9bb189de35f6ab, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/3359c9b1b6d84b58803024a47a72b895, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/c856148a5d6e4524abb63ac15fa55d29, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/3a058ce5ef66435d84ef6a82802131b5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/425d999521ac4b38aed2f1d0412884fc, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/ef1b43c56246419ebf9ba68487cee09d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/43eee8b7965e44fdb1f3ba1ece09966b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/275906c9d1f7405c9673bf5c47d1b612, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/beadb7e7e62247cc820e333c1884bdfd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/89e2af497ede47999a757566a46cb4fb, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/c43d0d1cfe74481bac89e56c0c2ebf9b] to archive 2024-11-17T21:26:33,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:26:33,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/40ed739b481f4795ad5ca352602d79f7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/40ed739b481f4795ad5ca352602d79f7 2024-11-17T21:26:33,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/2bdb7f5eff1c48099543da4dbcecca46 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/2bdb7f5eff1c48099543da4dbcecca46 2024-11-17T21:26:33,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/bf48abf8380a4478953a49ef232d9113 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/bf48abf8380a4478953a49ef232d9113 2024-11-17T21:26:33,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/a3b01a3359074b159b2261dfb4beabda to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/a3b01a3359074b159b2261dfb4beabda 2024-11-17T21:26:33,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/b58db119438a4582bd59a2578ad41e27 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/b58db119438a4582bd59a2578ad41e27 2024-11-17T21:26:33,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/254464c7ef7c4b14b62cea08b91e0be6 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/254464c7ef7c4b14b62cea08b91e0be6 2024-11-17T21:26:33,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/d234c8913b344a27a40866318545c523 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/d234c8913b344a27a40866318545c523 2024-11-17T21:26:33,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/43cdc12ba69c4b11bbe983abb6f94b3a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/43cdc12ba69c4b11bbe983abb6f94b3a 2024-11-17T21:26:33,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/39ec304604194e53b7a1b5f07329bd49 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/39ec304604194e53b7a1b5f07329bd49 2024-11-17T21:26:33,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/7068c193f8ae4c99b3ee14c7cc955b28 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/7068c193f8ae4c99b3ee14c7cc955b28 2024-11-17T21:26:33,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/841db4ef4e56493f979720ac3688f2c8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/841db4ef4e56493f979720ac3688f2c8 2024-11-17T21:26:33,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/4a76ae085aec4aad91de82ce72e476f4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/4a76ae085aec4aad91de82ce72e476f4 2024-11-17T21:26:33,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/54732f2c8d8f40e79aba7443dd6c27e1 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/54732f2c8d8f40e79aba7443dd6c27e1 2024-11-17T21:26:33,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/b789807916ea414d9e9bb189de35f6ab to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/b789807916ea414d9e9bb189de35f6ab 2024-11-17T21:26:33,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/3359c9b1b6d84b58803024a47a72b895 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/3359c9b1b6d84b58803024a47a72b895 2024-11-17T21:26:33,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/c856148a5d6e4524abb63ac15fa55d29 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/c856148a5d6e4524abb63ac15fa55d29 2024-11-17T21:26:33,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/3a058ce5ef66435d84ef6a82802131b5 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/3a058ce5ef66435d84ef6a82802131b5 2024-11-17T21:26:33,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/425d999521ac4b38aed2f1d0412884fc to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/425d999521ac4b38aed2f1d0412884fc 2024-11-17T21:26:33,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/ef1b43c56246419ebf9ba68487cee09d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/ef1b43c56246419ebf9ba68487cee09d 2024-11-17T21:26:33,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/43eee8b7965e44fdb1f3ba1ece09966b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/43eee8b7965e44fdb1f3ba1ece09966b 2024-11-17T21:26:33,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/275906c9d1f7405c9673bf5c47d1b612 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/275906c9d1f7405c9673bf5c47d1b612 2024-11-17T21:26:33,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/beadb7e7e62247cc820e333c1884bdfd to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/beadb7e7e62247cc820e333c1884bdfd 2024-11-17T21:26:33,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/89e2af497ede47999a757566a46cb4fb to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/89e2af497ede47999a757566a46cb4fb 2024-11-17T21:26:33,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/c43d0d1cfe74481bac89e56c0c2ebf9b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/c43d0d1cfe74481bac89e56c0c2ebf9b 2024-11-17T21:26:33,076 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/recovered.edits/374.seqid, newMaxSeqId=374, maxSeqId=1 2024-11-17T21:26:33,076 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. 2024-11-17T21:26:33,076 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 1e1006bdbe13128ffffb3f217a9e33ec: Waiting for close lock at 1731878791774Running coprocessor pre-close hooks at 1731878791774Disabling compacts and flushes for region at 1731878791774Disabling writes for close at 1731878791774Obtaining lock to block concurrent updates at 1731878791774Preparing flush snapshotting stores in 1e1006bdbe13128ffffb3f217a9e33ec at 1731878791774Finished memstore snapshotting TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731878791774Flushing stores of TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec. at 1731878791775 (+1 ms)Flushing 1e1006bdbe13128ffffb3f217a9e33ec/A: creating writer at 1731878791775Flushing 1e1006bdbe13128ffffb3f217a9e33ec/A: appending metadata at 1731878791777 (+2 ms)Flushing 1e1006bdbe13128ffffb3f217a9e33ec/A: closing flushed file at 1731878791777Flushing 1e1006bdbe13128ffffb3f217a9e33ec/B: creating writer at 1731878792184 (+407 ms)Flushing 1e1006bdbe13128ffffb3f217a9e33ec/B: appending metadata at 1731878792186 (+2 ms)Flushing 1e1006bdbe13128ffffb3f217a9e33ec/B: closing flushed file at 1731878792186Flushing 1e1006bdbe13128ffffb3f217a9e33ec/C: creating writer at 1731878792593 (+407 ms)Flushing 1e1006bdbe13128ffffb3f217a9e33ec/C: appending metadata at 1731878792595 (+2 ms)Flushing 1e1006bdbe13128ffffb3f217a9e33ec/C: closing flushed file at 1731878792595Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63147d3f: reopening flushed file at 1731878793003 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5183b74b: reopening flushed file at 1731878793007 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66c0ca75: reopening flushed file at 1731878793010 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1e1006bdbe13128ffffb3f217a9e33ec in 1240ms, sequenceid=371, compaction requested=true at 1731878793014 (+4 ms)Writing region close event to WAL at 1731878793074 (+60 ms)Running coprocessor post-close hooks at 1731878793076 (+2 ms)Closed at 1731878793076 2024-11-17T21:26:33,078 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:33,078 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=1e1006bdbe13128ffffb3f217a9e33ec, regionState=CLOSED 2024-11-17T21:26:33,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:26:33,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-17T21:26:33,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e1006bdbe13128ffffb3f217a9e33ec, server=a313eea8709e,46445,1731878642828 in 1.4590 sec 2024-11-17T21:26:33,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-17T21:26:33,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1006bdbe13128ffffb3f217a9e33ec, UNASSIGN in 1.4620 sec 2024-11-17T21:26:33,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-17T21:26:33,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4650 sec 2024-11-17T21:26:33,085 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878793085"}]},"ts":"1731878793085"} 2024-11-17T21:26:33,086 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T21:26:33,086 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T21:26:33,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4730 sec 2024-11-17T21:26:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T21:26:33,761 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:33,762 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T21:26:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:33,763 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T21:26:33,764 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:33,765 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:33,767 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/recovered.edits] 2024-11-17T21:26:33,769 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/811e6b66c7844d1db03ef0fb8c2822b4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/811e6b66c7844d1db03ef0fb8c2822b4 2024-11-17T21:26:33,769 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/a630fe2ea94141f7846009f2bb5c06e5 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/a630fe2ea94141f7846009f2bb5c06e5 2024-11-17T21:26:33,770 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/f176deccbb124c50a8a34886dec2bb57 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/A/f176deccbb124c50a8a34886dec2bb57 2024-11-17T21:26:33,771 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/bb56c77fbb6f4c009248e0b4b79b2cd7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/bb56c77fbb6f4c009248e0b4b79b2cd7 2024-11-17T21:26:33,772 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/d99d02d9f9404988bd519c65f655a60e to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/d99d02d9f9404988bd519c65f655a60e 2024-11-17T21:26:33,773 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/f34d547f6df94d58958d49cba7b8b9a9 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/B/f34d547f6df94d58958d49cba7b8b9a9 2024-11-17T21:26:33,774 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/1278839bc9534f258e71c1a7915f3bad to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/1278839bc9534f258e71c1a7915f3bad 2024-11-17T21:26:33,775 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/52eee623d7c64f6ca16e24ef985d0039 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/52eee623d7c64f6ca16e24ef985d0039 2024-11-17T21:26:33,775 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/e36fff532e9b419abfded0ca3eedad2f to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/C/e36fff532e9b419abfded0ca3eedad2f 2024-11-17T21:26:33,777 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/recovered.edits/374.seqid to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec/recovered.edits/374.seqid 2024-11-17T21:26:33,778 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/1e1006bdbe13128ffffb3f217a9e33ec 2024-11-17T21:26:33,778 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T21:26:33,780 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:33,782 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T21:26:33,784 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T21:26:33,785 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:33,785 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T21:26:33,785 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731878793785"}]},"ts":"9223372036854775807"} 2024-11-17T21:26:33,787 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T21:26:33,787 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1e1006bdbe13128ffffb3f217a9e33ec, NAME => 'TestAcidGuarantees,,1731878765803.1e1006bdbe13128ffffb3f217a9e33ec.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T21:26:33,787 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T21:26:33,787 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731878793787"}]},"ts":"9223372036854775807"} 2024-11-17T21:26:33,788 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T21:26:33,789 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:33,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 27 msec 2024-11-17T21:26:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T21:26:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T21:26:33,873 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:33,878 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237 (was 237), OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=469 (was 521), ProcessCount=11 (was 11), AvailableMemoryMB=8001 (was 8149) 2024-11-17T21:26:33,886 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=469, ProcessCount=11, AvailableMemoryMB=8001 2024-11-17T21:26:33,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T21:26:33,888 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T21:26:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:33,889 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T21:26:33,889 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:33,889 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-17T21:26:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T21:26:33,890 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T21:26:33,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742354_1530 (size=959) 2024-11-17T21:26:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T21:26:34,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T21:26:34,296 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249 2024-11-17T21:26:34,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742355_1531 (size=53) 2024-11-17T21:26:34,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T21:26:34,701 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:26:34,701 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a21acdaa0aabfb075e55b063d1c8e884, disabling compactions & flushes 2024-11-17T21:26:34,701 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:34,701 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:34,701 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. after waiting 0 ms 2024-11-17T21:26:34,701 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:34,701 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:34,701 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a21acdaa0aabfb075e55b063d1c8e884: Waiting for close lock at 1731878794701Disabling compacts and flushes for region at 1731878794701Disabling writes for close at 1731878794701Writing region close event to WAL at 1731878794701Closed at 1731878794701 2024-11-17T21:26:34,702 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T21:26:34,703 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731878794702"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731878794702"}]},"ts":"1731878794702"} 2024-11-17T21:26:34,704 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T21:26:34,705 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T21:26:34,705 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878794705"}]},"ts":"1731878794705"} 2024-11-17T21:26:34,706 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T21:26:34,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a21acdaa0aabfb075e55b063d1c8e884, ASSIGN}] 2024-11-17T21:26:34,707 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a21acdaa0aabfb075e55b063d1c8e884, ASSIGN 2024-11-17T21:26:34,708 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a21acdaa0aabfb075e55b063d1c8e884, ASSIGN; state=OFFLINE, location=a313eea8709e,46445,1731878642828; forceNewPlan=false, retain=false 2024-11-17T21:26:34,859 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=a21acdaa0aabfb075e55b063d1c8e884, regionState=OPENING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:26:34,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a21acdaa0aabfb075e55b063d1c8e884, ASSIGN because future has completed 2024-11-17T21:26:34,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:26:35,015 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:35,015 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} 2024-11-17T21:26:35,015 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:35,015 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:26:35,015 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:35,015 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:35,016 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:35,017 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a21acdaa0aabfb075e55b063d1c8e884 columnFamilyName A 2024-11-17T21:26:35,017 DEBUG [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:35,017 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] regionserver.HStore(327): Store=a21acdaa0aabfb075e55b063d1c8e884/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:26:35,017 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:35,018 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a21acdaa0aabfb075e55b063d1c8e884 columnFamilyName B 2024-11-17T21:26:35,018 DEBUG [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:35,018 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] regionserver.HStore(327): Store=a21acdaa0aabfb075e55b063d1c8e884/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:26:35,018 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:35,019 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a21acdaa0aabfb075e55b063d1c8e884 columnFamilyName C 2024-11-17T21:26:35,019 DEBUG [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:35,019 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] regionserver.HStore(327): Store=a21acdaa0aabfb075e55b063d1c8e884/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:26:35,020 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:35,020 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:35,020 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:35,021 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:35,021 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:35,022 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T21:26:35,023 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:35,025 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T21:26:35,025 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened a21acdaa0aabfb075e55b063d1c8e884; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61871375, jitterRate=-0.07804466784000397}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T21:26:35,025 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:35,026 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for a21acdaa0aabfb075e55b063d1c8e884: Running coprocessor pre-open hook at 1731878795015Writing region info on filesystem at 1731878795015Initializing all the Stores at 1731878795016 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878795016Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878795016Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878795016Cleaning up temporary data from old regions at 1731878795021 (+5 ms)Running coprocessor post-open hooks at 1731878795025 (+4 ms)Region opened successfully at 1731878795026 (+1 ms) 2024-11-17T21:26:35,027 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., pid=144, masterSystemTime=1731878795012 2024-11-17T21:26:35,028 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:35,028 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:35,029 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=a21acdaa0aabfb075e55b063d1c8e884, regionState=OPEN, openSeqNum=2, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:26:35,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:26:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T21:26:35,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-17T21:26:35,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 in 170 msec 2024-11-17T21:26:35,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-17T21:26:35,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a21acdaa0aabfb075e55b063d1c8e884, ASSIGN in 325 msec 2024-11-17T21:26:35,033 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T21:26:35,034 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878795034"}]},"ts":"1731878795034"} 2024-11-17T21:26:35,035 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T21:26:35,035 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T21:26:35,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-17T21:26:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T21:26:36,041 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:36,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f71979f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,042 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:36,042 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:36,043 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:36,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:36,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:36,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@263dbaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:36,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:36,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:36,045 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:36,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ecd1b84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,045 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:36,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:36,048 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T21:26:36,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is a313eea8709e,43623,1731878641945 2024-11-17T21:26:36,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@21c1da46 2024-11-17T21:26:36,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T21:26:36,049 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T21:26:36,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T21:26:36,050 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T21:26:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T21:26:36,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742356_1532 (size=995) 2024-11-17T21:26:36,462 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-17T21:26:36,462 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-17T21:26:36,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T21:26:36,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a21acdaa0aabfb075e55b063d1c8e884, REOPEN/MOVE}] 2024-11-17T21:26:36,466 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a21acdaa0aabfb075e55b063d1c8e884, REOPEN/MOVE 2024-11-17T21:26:36,467 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=a21acdaa0aabfb075e55b063d1c8e884, regionState=CLOSING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:26:36,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a21acdaa0aabfb075e55b063d1c8e884, REOPEN/MOVE because future has completed 2024-11-17T21:26:36,468 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T21:26:36,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:26:36,620 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:36,620 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T21:26:36,620 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing a21acdaa0aabfb075e55b063d1c8e884, disabling compactions & flushes 2024-11-17T21:26:36,620 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:36,620 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:36,620 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. after waiting 0 ms 2024-11-17T21:26:36,620 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:36,623 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T21:26:36,624 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:36,624 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for a21acdaa0aabfb075e55b063d1c8e884: Waiting for close lock at 1731878796620Running coprocessor pre-close hooks at 1731878796620Disabling compacts and flushes for region at 1731878796620Disabling writes for close at 1731878796620Writing region close event to WAL at 1731878796621 (+1 ms)Running coprocessor post-close hooks at 1731878796624 (+3 ms)Closed at 1731878796624 2024-11-17T21:26:36,624 WARN [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: a21acdaa0aabfb075e55b063d1c8e884 to self. 2024-11-17T21:26:36,625 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:36,626 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=a21acdaa0aabfb075e55b063d1c8e884, regionState=CLOSED 2024-11-17T21:26:36,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:26:36,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-17T21:26:36,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 in 159 msec 2024-11-17T21:26:36,629 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a21acdaa0aabfb075e55b063d1c8e884, REOPEN/MOVE; state=CLOSED, location=a313eea8709e,46445,1731878642828; forceNewPlan=false, retain=true 2024-11-17T21:26:36,780 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=a21acdaa0aabfb075e55b063d1c8e884, regionState=OPENING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:26:36,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a21acdaa0aabfb075e55b063d1c8e884, REOPEN/MOVE because future has completed 2024-11-17T21:26:36,781 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:26:36,935 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:36,935 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} 2024-11-17T21:26:36,935 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:36,935 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T21:26:36,936 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:36,936 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:36,936 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:36,937 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a21acdaa0aabfb075e55b063d1c8e884 columnFamilyName A 2024-11-17T21:26:36,938 DEBUG [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:36,938 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] regionserver.HStore(327): Store=a21acdaa0aabfb075e55b063d1c8e884/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:26:36,938 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:36,939 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a21acdaa0aabfb075e55b063d1c8e884 columnFamilyName B 2024-11-17T21:26:36,939 DEBUG [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:36,939 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] regionserver.HStore(327): Store=a21acdaa0aabfb075e55b063d1c8e884/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:26:36,939 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:36,940 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a21acdaa0aabfb075e55b063d1c8e884 columnFamilyName C 2024-11-17T21:26:36,940 DEBUG [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:36,940 INFO [StoreOpener-a21acdaa0aabfb075e55b063d1c8e884-1 {}] regionserver.HStore(327): Store=a21acdaa0aabfb075e55b063d1c8e884/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T21:26:36,940 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:36,940 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:36,941 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:36,942 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:36,942 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:36,942 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T21:26:36,943 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:36,943 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened a21acdaa0aabfb075e55b063d1c8e884; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64115500, jitterRate=-0.044604599475860596}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T21:26:36,943 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:36,944 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for a21acdaa0aabfb075e55b063d1c8e884: Running coprocessor pre-open hook at 1731878796936Writing region info on filesystem at 1731878796936Initializing all the Stores at 1731878796936Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878796936Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878796936Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731878796936Cleaning up temporary data from old regions at 1731878796942 (+6 ms)Running coprocessor post-open hooks at 1731878796943 (+1 ms)Region opened successfully at 1731878796944 (+1 ms) 2024-11-17T21:26:36,945 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., pid=149, masterSystemTime=1731878796933 2024-11-17T21:26:36,946 DEBUG [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:36,946 INFO [RS_OPEN_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:36,946 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=a21acdaa0aabfb075e55b063d1c8e884, regionState=OPEN, openSeqNum=5, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:26:36,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:26:36,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-17T21:26:36,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 in 167 msec 2024-11-17T21:26:36,950 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-17T21:26:36,950 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a21acdaa0aabfb075e55b063d1c8e884, REOPEN/MOVE in 484 msec 2024-11-17T21:26:36,952 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-17T21:26:36,952 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 486 msec 2024-11-17T21:26:36,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 901 msec 2024-11-17T21:26:36,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-17T21:26:36,954 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:36,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7663d9ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,954 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:36,955 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:36,955 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:36,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:36,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:36,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7882b31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:36,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:36,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:36,956 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@384c2bb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3876a305, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,957 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:36,957 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:36,958 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:36,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:36,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:36,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b43f031, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:36,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:36,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:36,959 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4da4e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6266891e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,960 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:36,960 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:36,960 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:36,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:36,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:36,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50368329, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:36,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:36,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:36,961 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f19481, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79538ae3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,962 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:36,962 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:36,963 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1abf02f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:36,964 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c26f838, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4601a12f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:36,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:36,967 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5768cf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:36,968 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ce01fcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d16a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:36,969 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:36,970 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:36,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:36,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:36,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b41f23f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:36,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:36,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:36,971 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@759d27bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16487325, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:36,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:36,972 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11496492, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:36,973 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:36,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa51c0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5129a55e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,974 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:36,974 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:36,975 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f44a943, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:36,976 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:36,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d651f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79f0ee4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:36,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:36,977 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:36,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:36,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:36,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@306f11f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:36,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:36,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:36,978 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@565eac68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cf3480a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,979 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a313eea8709e,43623,-1 for getting cluster id 2024-11-17T21:26:36,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T21:26:36,980 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8f8173f4-7420-463b-8a7b-f3fcd9daf410' 2024-11-17T21:26:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T21:26:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8f8173f4-7420-463b-8a7b-f3fcd9daf410" 2024-11-17T21:26:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@211ead27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a313eea8709e,43623,-1] 2024-11-17T21:26:36,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T21:26:36,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:36,981 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T21:26:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1901965c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T21:26:36,982 DEBUG [Thread-2380 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:36,983 DEBUG [Thread-2376 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:36,983 DEBUG [Thread-2393 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:36,983 DEBUG [Thread-2387 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:36,983 DEBUG [Thread-2384 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:36,984 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:36,985 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:36,985 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:36,985 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:36,985 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:36,986 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:36,986 DEBUG [Thread-2391 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:36,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:26:36,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:26:36,987 DEBUG [Thread-2389 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-17T21:26:36,987 DEBUG [Thread-2382 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:36,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:36,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:36,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:26:36,988 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:36,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:36,989 DEBUG [Thread-2395 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:36,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:26:36,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:36,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:36,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:36,989 DEBUG [Thread-2378 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T21:26:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:26:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:36,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T21:26:36,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:26:36,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:36,992 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:36,993 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:26:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:26:36,994 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a313eea8709e,46445,1731878642828, seqNum=-1] 2024-11-17T21:26:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T21:26:36,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:26:36,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T21:26:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5] 2024-11-17T21:26:37,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:37,002 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:37,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:26:37,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c6fdf8f1b292403dbcdd0fb577f970b9_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878797001/Put/seqid=0 2024-11-17T21:26:37,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878857033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878857034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878857034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878857035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52140 deadline: 1731878857039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742357_1533 (size=12154) 2024-11-17T21:26:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T21:26:37,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878857140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878857140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878857141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T21:26:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:37,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:37,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52140 deadline: 1731878857151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878857151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T21:26:37,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:37,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:37,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:37,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:37,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T21:26:37,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878857351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878857351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878857351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878857361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52140 deadline: 1731878857361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,450 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:37,453 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c6fdf8f1b292403dbcdd0fb577f970b9_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c6fdf8f1b292403dbcdd0fb577f970b9_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:37,453 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/cd1b5c760b504d7486d5e30822011fe5, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:37,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T21:26:37,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:37,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:37,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:37,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/cd1b5c760b504d7486d5e30822011fe5 is 175, key is test_row_0/A:col10/1731878797001/Put/seqid=0 2024-11-17T21:26:37,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:37,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:37,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742358_1534 (size=30955) 2024-11-17T21:26:37,481 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/cd1b5c760b504d7486d5e30822011fe5 2024-11-17T21:26:37,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/bab509df0d834661a52d047498867e9b is 50, key is test_row_0/B:col10/1731878797001/Put/seqid=0 2024-11-17T21:26:37,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742359_1535 (size=12001) 2024-11-17T21:26:37,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/bab509df0d834661a52d047498867e9b 2024-11-17T21:26:37,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/23d6b0bd81cc4ba2b2e7fb45342255c6 is 50, key is test_row_0/C:col10/1731878797001/Put/seqid=0 2024-11-17T21:26:37,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742360_1536 (size=12001) 2024-11-17T21:26:37,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/23d6b0bd81cc4ba2b2e7fb45342255c6 2024-11-17T21:26:37,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/cd1b5c760b504d7486d5e30822011fe5 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/cd1b5c760b504d7486d5e30822011fe5 2024-11-17T21:26:37,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/cd1b5c760b504d7486d5e30822011fe5, entries=150, sequenceid=16, filesize=30.2 K 2024-11-17T21:26:37,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/bab509df0d834661a52d047498867e9b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/bab509df0d834661a52d047498867e9b 2024-11-17T21:26:37,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/bab509df0d834661a52d047498867e9b, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T21:26:37,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/23d6b0bd81cc4ba2b2e7fb45342255c6 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/23d6b0bd81cc4ba2b2e7fb45342255c6 2024-11-17T21:26:37,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/23d6b0bd81cc4ba2b2e7fb45342255c6, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T21:26:37,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a21acdaa0aabfb075e55b063d1c8e884 in 563ms, sequenceid=16, compaction requested=false 2024-11-17T21:26:37,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:37,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T21:26:37,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:37,607 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T21:26:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T21:26:37,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117dfc7b0688dab48d1aee48b241a88ae7c_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878797034/Put/seqid=0 2024-11-17T21:26:37,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742361_1537 (size=12154) 2024-11-17T21:26:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:37,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:37,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878857670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52140 deadline: 1731878857671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878857671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878857671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878857672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878857781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878857781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878857781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878857991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878857991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878857991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:38,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:38,041 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117dfc7b0688dab48d1aee48b241a88ae7c_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117dfc7b0688dab48d1aee48b241a88ae7c_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:38,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/dc0ed2d2f5dd4789b6523b70e8f9f7f8, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:38,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/dc0ed2d2f5dd4789b6523b70e8f9f7f8 is 175, key is test_row_0/A:col10/1731878797034/Put/seqid=0 2024-11-17T21:26:38,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742362_1538 (size=30955) 2024-11-17T21:26:38,048 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/dc0ed2d2f5dd4789b6523b70e8f9f7f8 2024-11-17T21:26:38,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/0c2f07ef4ec94b5db2dcfcf340b846ed is 50, key is test_row_0/B:col10/1731878797034/Put/seqid=0 2024-11-17T21:26:38,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742363_1539 (size=12001) 2024-11-17T21:26:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T21:26:38,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52140 deadline: 1731878858191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:38,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878858191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:38,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878858301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878858301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878858302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:38,470 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/0c2f07ef4ec94b5db2dcfcf340b846ed 2024-11-17T21:26:38,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/530c5ca2103d4e6298fd34bec4e38947 is 50, key is test_row_0/C:col10/1731878797034/Put/seqid=0 2024-11-17T21:26:38,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742364_1540 (size=12001) 2024-11-17T21:26:38,485 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/530c5ca2103d4e6298fd34bec4e38947 2024-11-17T21:26:38,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/dc0ed2d2f5dd4789b6523b70e8f9f7f8 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dc0ed2d2f5dd4789b6523b70e8f9f7f8 2024-11-17T21:26:38,499 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dc0ed2d2f5dd4789b6523b70e8f9f7f8, entries=150, sequenceid=40, filesize=30.2 K 2024-11-17T21:26:38,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/0c2f07ef4ec94b5db2dcfcf340b846ed as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/0c2f07ef4ec94b5db2dcfcf340b846ed 2024-11-17T21:26:38,503 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/0c2f07ef4ec94b5db2dcfcf340b846ed, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T21:26:38,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/530c5ca2103d4e6298fd34bec4e38947 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/530c5ca2103d4e6298fd34bec4e38947 2024-11-17T21:26:38,506 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/530c5ca2103d4e6298fd34bec4e38947, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T21:26:38,507 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for a21acdaa0aabfb075e55b063d1c8e884 in 900ms, sequenceid=40, compaction requested=false 2024-11-17T21:26:38,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:38,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:38,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-17T21:26:38,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-17T21:26:38,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-17T21:26:38,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5700 sec 2024-11-17T21:26:38,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.5800 sec 2024-11-17T21:26:38,604 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T21:26:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:38,814 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:38,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T21:26:38,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174684de26870c4add90750ef0c7aa9300_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878798813/Put/seqid=0 2024-11-17T21:26:38,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742365_1541 (size=14594) 2024-11-17T21:26:38,826 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:38,828 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174684de26870c4add90750ef0c7aa9300_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174684de26870c4add90750ef0c7aa9300_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:38,829 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/dcc235bb8e6c4cffaf1cb8885b7bb6f0, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:38,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/dcc235bb8e6c4cffaf1cb8885b7bb6f0 is 175, key is test_row_0/A:col10/1731878798813/Put/seqid=0 2024-11-17T21:26:38,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878858844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:38,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878858845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:38,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878858844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:38,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742366_1542 (size=39549) 2024-11-17T21:26:38,860 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/dcc235bb8e6c4cffaf1cb8885b7bb6f0 2024-11-17T21:26:38,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/51e592e43b9c4d41a8fa628806c55ea9 is 50, key is test_row_0/B:col10/1731878798813/Put/seqid=0 2024-11-17T21:26:38,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742367_1543 (size=12001) 2024-11-17T21:26:38,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878858950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:38,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878858951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:38,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:38,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878858961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T21:26:39,141 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:39,142 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-17T21:26:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T21:26:39,143 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:39,144 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:39,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:39,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878859160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:39,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878859161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:39,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878859171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:39,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878859201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:39,203 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in a21acdaa0aabfb075e55b063d1c8e884 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:39,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:39,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:39,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:39,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52140 deadline: 1731878859210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:39,214 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_2' in a21acdaa0aabfb075e55b063d1c8e884 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:39,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:39,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:39,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T21:26:39,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/51e592e43b9c4d41a8fa628806c55ea9 2024-11-17T21:26:39,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/e219dcde6a3f442ca757d618a180c6ec is 50, key is test_row_0/C:col10/1731878798813/Put/seqid=0 2024-11-17T21:26:39,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742368_1544 (size=12001) 2024-11-17T21:26:39,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T21:26:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:39,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:39,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T21:26:39,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:39,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:39,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:39,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:39,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T21:26:39,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878859470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:39,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878859470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:39,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878859481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:39,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:39,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:39,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:39,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T21:26:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:39,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:39,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:39,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/e219dcde6a3f442ca757d618a180c6ec 2024-11-17T21:26:39,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/dcc235bb8e6c4cffaf1cb8885b7bb6f0 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dcc235bb8e6c4cffaf1cb8885b7bb6f0 2024-11-17T21:26:39,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dcc235bb8e6c4cffaf1cb8885b7bb6f0, entries=200, sequenceid=55, filesize=38.6 K 2024-11-17T21:26:39,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/51e592e43b9c4d41a8fa628806c55ea9 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/51e592e43b9c4d41a8fa628806c55ea9 2024-11-17T21:26:39,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/51e592e43b9c4d41a8fa628806c55ea9, entries=150, sequenceid=55, filesize=11.7 K 2024-11-17T21:26:39,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/e219dcde6a3f442ca757d618a180c6ec as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/e219dcde6a3f442ca757d618a180c6ec 2024-11-17T21:26:39,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/e219dcde6a3f442ca757d618a180c6ec, entries=150, sequenceid=55, filesize=11.7 K 2024-11-17T21:26:39,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for a21acdaa0aabfb075e55b063d1c8e884 in 906ms, sequenceid=55, compaction requested=true 2024-11-17T21:26:39,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:39,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:26:39,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:39,720 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:39,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:26:39,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:39,720 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:39,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:26:39,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:39,721 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:39,721 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/B is initiating minor compaction (all files) 2024-11-17T21:26:39,721 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/B in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:39,721 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/bab509df0d834661a52d047498867e9b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/0c2f07ef4ec94b5db2dcfcf340b846ed, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/51e592e43b9c4d41a8fa628806c55ea9] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=35.2 K 2024-11-17T21:26:39,721 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:39,721 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/A is initiating minor compaction (all files) 2024-11-17T21:26:39,721 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/A in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:39,722 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/cd1b5c760b504d7486d5e30822011fe5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dc0ed2d2f5dd4789b6523b70e8f9f7f8, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dcc235bb8e6c4cffaf1cb8885b7bb6f0] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=99.1 K 2024-11-17T21:26:39,722 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:39,722 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/cd1b5c760b504d7486d5e30822011fe5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dc0ed2d2f5dd4789b6523b70e8f9f7f8, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dcc235bb8e6c4cffaf1cb8885b7bb6f0] 2024-11-17T21:26:39,722 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd1b5c760b504d7486d5e30822011fe5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731878796996 2024-11-17T21:26:39,722 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting bab509df0d834661a52d047498867e9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731878796996 2024-11-17T21:26:39,722 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c2f07ef4ec94b5db2dcfcf340b846ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731878797033 2024-11-17T21:26:39,722 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc0ed2d2f5dd4789b6523b70e8f9f7f8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731878797033 2024-11-17T21:26:39,722 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 51e592e43b9c4d41a8fa628806c55ea9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731878798813 2024-11-17T21:26:39,723 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcc235bb8e6c4cffaf1cb8885b7bb6f0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731878797671 2024-11-17T21:26:39,728 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#B#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:39,729 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/1f711e11c3854e03a19ec87476db3188 is 50, key is test_row_0/B:col10/1731878798813/Put/seqid=0 2024-11-17T21:26:39,749 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:39,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T21:26:39,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:39,754 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T21:26:39,765 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117c6817d3f8b284207a807d400b5d18c2c_a21acdaa0aabfb075e55b063d1c8e884 store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:39,767 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117c6817d3f8b284207a807d400b5d18c2c_a21acdaa0aabfb075e55b063d1c8e884, store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:39,767 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c6817d3f8b284207a807d400b5d18c2c_a21acdaa0aabfb075e55b063d1c8e884 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T21:26:39,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742369_1545 (size=12104) 2024-11-17T21:26:39,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111787d78de9ecc44a89b964d6d60060d959_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878798843/Put/seqid=0 2024-11-17T21:26:39,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742370_1546 (size=4469) 2024-11-17T21:26:39,808 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#A#compaction#462 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:39,809 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/db67ad073c9f4f54b7b4a44675bf303f is 175, key is test_row_0/A:col10/1731878798813/Put/seqid=0 2024-11-17T21:26:39,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742371_1547 (size=12154) 2024-11-17T21:26:39,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:39,851 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111787d78de9ecc44a89b964d6d60060d959_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111787d78de9ecc44a89b964d6d60060d959_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:39,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/3586f00f4aa144da9f3bd75f8e4bd018, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:39,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/3586f00f4aa144da9f3bd75f8e4bd018 is 175, key is test_row_0/A:col10/1731878798843/Put/seqid=0 2024-11-17T21:26:39,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742372_1548 (size=31058) 2024-11-17T21:26:39,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742373_1549 (size=30955) 2024-11-17T21:26:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:39,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:40,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878860000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:40,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:40,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878860003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:40,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:40,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878860004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:40,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:40,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:40,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878860111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:40,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:40,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878860111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:40,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878860111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:40,180 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/1f711e11c3854e03a19ec87476db3188 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/1f711e11c3854e03a19ec87476db3188 2024-11-17T21:26:40,183 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/B of a21acdaa0aabfb075e55b063d1c8e884 into 1f711e11c3854e03a19ec87476db3188(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:40,183 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:40,183 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/B, priority=13, startTime=1731878799720; duration=0sec 2024-11-17T21:26:40,183 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:40,183 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:B 2024-11-17T21:26:40,183 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:40,184 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:40,184 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/C is initiating minor compaction (all files) 2024-11-17T21:26:40,184 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/C in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:40,184 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/23d6b0bd81cc4ba2b2e7fb45342255c6, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/530c5ca2103d4e6298fd34bec4e38947, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/e219dcde6a3f442ca757d618a180c6ec] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=35.2 K 2024-11-17T21:26:40,184 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 23d6b0bd81cc4ba2b2e7fb45342255c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731878796996 2024-11-17T21:26:40,185 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 530c5ca2103d4e6298fd34bec4e38947, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731878797033 2024-11-17T21:26:40,185 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting e219dcde6a3f442ca757d618a180c6ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731878798813 2024-11-17T21:26:40,190 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#C#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:40,190 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/0a384390bcbe4dd9bd9c8166641be8fe is 50, key is test_row_0/C:col10/1731878798813/Put/seqid=0 2024-11-17T21:26:40,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742374_1550 (size=12104) 2024-11-17T21:26:40,261 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/db67ad073c9f4f54b7b4a44675bf303f as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/db67ad073c9f4f54b7b4a44675bf303f 2024-11-17T21:26:40,264 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/A of a21acdaa0aabfb075e55b063d1c8e884 into db67ad073c9f4f54b7b4a44675bf303f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:40,264 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:40,264 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/A, priority=13, startTime=1731878799720; duration=0sec 2024-11-17T21:26:40,264 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:40,264 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:A 2024-11-17T21:26:40,278 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/3586f00f4aa144da9f3bd75f8e4bd018 2024-11-17T21:26:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T21:26:40,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/84af635efa0a4c3bb597920716a8814d is 50, key is test_row_0/B:col10/1731878798843/Put/seqid=0 2024-11-17T21:26:40,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742375_1551 (size=12001) 2024-11-17T21:26:40,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878860321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878860321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:40,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878860321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:40,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:40,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:40,605 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/0a384390bcbe4dd9bd9c8166641be8fe as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/0a384390bcbe4dd9bd9c8166641be8fe 2024-11-17T21:26:40,609 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/C of a21acdaa0aabfb075e55b063d1c8e884 into 0a384390bcbe4dd9bd9c8166641be8fe(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:40,609 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:40,609 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/C, priority=13, startTime=1731878799720; duration=0sec 2024-11-17T21:26:40,609 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:40,609 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:C 2024-11-17T21:26:40,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:40,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878860631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:40,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878860631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:40,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878860631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:40,688 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/84af635efa0a4c3bb597920716a8814d 2024-11-17T21:26:40,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/8732b6c6b96e4aa7964266e61d1771ee is 50, key is test_row_0/C:col10/1731878798843/Put/seqid=0 2024-11-17T21:26:40,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742376_1552 (size=12001) 2024-11-17T21:26:40,697 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/8732b6c6b96e4aa7964266e61d1771ee 2024-11-17T21:26:40,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/3586f00f4aa144da9f3bd75f8e4bd018 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/3586f00f4aa144da9f3bd75f8e4bd018 2024-11-17T21:26:40,707 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/3586f00f4aa144da9f3bd75f8e4bd018, entries=150, sequenceid=76, filesize=30.2 K 2024-11-17T21:26:40,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/84af635efa0a4c3bb597920716a8814d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/84af635efa0a4c3bb597920716a8814d 2024-11-17T21:26:40,711 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/84af635efa0a4c3bb597920716a8814d, entries=150, sequenceid=76, filesize=11.7 K 2024-11-17T21:26:40,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/8732b6c6b96e4aa7964266e61d1771ee as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/8732b6c6b96e4aa7964266e61d1771ee 2024-11-17T21:26:40,719 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/8732b6c6b96e4aa7964266e61d1771ee, entries=150, sequenceid=76, filesize=11.7 K 2024-11-17T21:26:40,721 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for a21acdaa0aabfb075e55b063d1c8e884 in 966ms, sequenceid=76, compaction requested=false 2024-11-17T21:26:40,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:40,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:40,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-17T21:26:40,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-17T21:26:40,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-17T21:26:40,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5780 sec 2024-11-17T21:26:40,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.5820 sec 2024-11-17T21:26:41,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:41,143 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:41,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T21:26:41,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f05189b9b3474279ad30b98d78718767_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878799999/Put/seqid=0 2024-11-17T21:26:41,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742377_1553 (size=14594) 2024-11-17T21:26:41,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878861168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:41,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878861168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:41,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:41,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878861171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:41,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:41,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878861221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:41,224 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in a21acdaa0aabfb075e55b063d1c8e884 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:41,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:41,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:41,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52140 deadline: 1731878861230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:41,233 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_2' in a21acdaa0aabfb075e55b063d1c8e884 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:41,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:41,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878861281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:41,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:41,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878861281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:41,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:41,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878861281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:41,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:41,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:41,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T21:26:41,291 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:41,291 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-17T21:26:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T21:26:41,293 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:41,293 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:41,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T21:26:41,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T21:26:41,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:41,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:41,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:41,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:41,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:41,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878861491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:41,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:41,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878861491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:41,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:41,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878861500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:41,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:41,557 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:41,560 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f05189b9b3474279ad30b98d78718767_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f05189b9b3474279ad30b98d78718767_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:41,560 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/f969075e0f5f4d3c9caf44eeedbf7f14, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:41,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/f969075e0f5f4d3c9caf44eeedbf7f14 is 175, key is test_row_0/A:col10/1731878799999/Put/seqid=0 2024-11-17T21:26:41,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742378_1554 (size=39549) 2024-11-17T21:26:41,566 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/f969075e0f5f4d3c9caf44eeedbf7f14 2024-11-17T21:26:41,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/db07635a8020470b844701e3f9f01685 is 50, key is test_row_0/B:col10/1731878799999/Put/seqid=0 2024-11-17T21:26:41,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742379_1555 (size=12001) 2024-11-17T21:26:41,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T21:26:41,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:41,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:41,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:41,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:41,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:41,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:41,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T21:26:41,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T21:26:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:41,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:41,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878861801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878861801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:41,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878861811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:41,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:41,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:41,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T21:26:41,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:41,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:41,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:41,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:41,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T21:26:41,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/db07635a8020470b844701e3f9f01685 2024-11-17T21:26:41,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/ea3ce61f2f7a40e394c1e938e7f1be47 is 50, key is test_row_0/C:col10/1731878799999/Put/seqid=0 2024-11-17T21:26:41,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742380_1556 (size=12001) 2024-11-17T21:26:42,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T21:26:42,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:42,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:42,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:42,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:42,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:42,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T21:26:42,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:42,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:42,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:42,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:42,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:42,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878862311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:42,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878862311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:42,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878862320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:42,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T21:26:42,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:42,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:42,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:42,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:42,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:42,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:42,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/ea3ce61f2f7a40e394c1e938e7f1be47 2024-11-17T21:26:42,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/f969075e0f5f4d3c9caf44eeedbf7f14 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/f969075e0f5f4d3c9caf44eeedbf7f14 2024-11-17T21:26:42,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/f969075e0f5f4d3c9caf44eeedbf7f14, entries=200, sequenceid=95, filesize=38.6 K 2024-11-17T21:26:42,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/db07635a8020470b844701e3f9f01685 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/db07635a8020470b844701e3f9f01685 2024-11-17T21:26:42,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/db07635a8020470b844701e3f9f01685, entries=150, sequenceid=95, filesize=11.7 K 2024-11-17T21:26:42,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/ea3ce61f2f7a40e394c1e938e7f1be47 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/ea3ce61f2f7a40e394c1e938e7f1be47 2024-11-17T21:26:42,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/ea3ce61f2f7a40e394c1e938e7f1be47, entries=150, sequenceid=95, filesize=11.7 K 2024-11-17T21:26:42,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for a21acdaa0aabfb075e55b063d1c8e884 in 1263ms, sequenceid=95, compaction requested=true 2024-11-17T21:26:42,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:42,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:26:42,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:42,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:26:42,406 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:42,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:42,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:26:42,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:42,406 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:42,407 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:42,407 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:42,407 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/B is initiating minor compaction (all files) 2024-11-17T21:26:42,407 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/A is initiating minor compaction (all files) 2024-11-17T21:26:42,407 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/B in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:42,407 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/A in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:42,407 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/1f711e11c3854e03a19ec87476db3188, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/84af635efa0a4c3bb597920716a8814d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/db07635a8020470b844701e3f9f01685] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=35.3 K 2024-11-17T21:26:42,407 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/db67ad073c9f4f54b7b4a44675bf303f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/3586f00f4aa144da9f3bd75f8e4bd018, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/f969075e0f5f4d3c9caf44eeedbf7f14] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=99.2 K 2024-11-17T21:26:42,407 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:42,407 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/db67ad073c9f4f54b7b4a44675bf303f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/3586f00f4aa144da9f3bd75f8e4bd018, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/f969075e0f5f4d3c9caf44eeedbf7f14] 2024-11-17T21:26:42,408 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting db67ad073c9f4f54b7b4a44675bf303f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731878798813 2024-11-17T21:26:42,408 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f711e11c3854e03a19ec87476db3188, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731878798813 2024-11-17T21:26:42,408 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3586f00f4aa144da9f3bd75f8e4bd018, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731878798843 2024-11-17T21:26:42,408 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 84af635efa0a4c3bb597920716a8814d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731878798843 2024-11-17T21:26:42,408 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting db07635a8020470b844701e3f9f01685, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731878799999 2024-11-17T21:26:42,408 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting f969075e0f5f4d3c9caf44eeedbf7f14, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731878799999 2024-11-17T21:26:42,412 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:42,413 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#B#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:42,413 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/4235733e698046fc83174f8e9db684d0 is 50, key is test_row_0/B:col10/1731878799999/Put/seqid=0 2024-11-17T21:26:42,415 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117bb38bf43ce3847f18315c9540627ebbc_a21acdaa0aabfb075e55b063d1c8e884 store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:42,417 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117bb38bf43ce3847f18315c9540627ebbc_a21acdaa0aabfb075e55b063d1c8e884, store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:42,417 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bb38bf43ce3847f18315c9540627ebbc_a21acdaa0aabfb075e55b063d1c8e884 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:42,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742381_1557 (size=12207) 2024-11-17T21:26:42,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742382_1558 (size=4469) 2024-11-17T21:26:42,429 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/4235733e698046fc83174f8e9db684d0 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/4235733e698046fc83174f8e9db684d0 2024-11-17T21:26:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T21:26:42,433 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#A#compaction#471 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:42,433 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/B of a21acdaa0aabfb075e55b063d1c8e884 into 4235733e698046fc83174f8e9db684d0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:42,433 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:42,433 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/B, priority=13, startTime=1731878802406; duration=0sec 2024-11-17T21:26:42,433 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:42,433 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:B 2024-11-17T21:26:42,433 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:42,434 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/c484faa8dee04bf08eefb06f6f16c03d is 175, key is test_row_0/A:col10/1731878799999/Put/seqid=0 2024-11-17T21:26:42,435 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:42,435 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/C is initiating minor compaction (all files) 2024-11-17T21:26:42,435 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/C in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:42,435 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/0a384390bcbe4dd9bd9c8166641be8fe, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/8732b6c6b96e4aa7964266e61d1771ee, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/ea3ce61f2f7a40e394c1e938e7f1be47] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=35.3 K 2024-11-17T21:26:42,435 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a384390bcbe4dd9bd9c8166641be8fe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731878798813 2024-11-17T21:26:42,436 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 8732b6c6b96e4aa7964266e61d1771ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731878798843 2024-11-17T21:26:42,436 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting ea3ce61f2f7a40e394c1e938e7f1be47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731878799999 2024-11-17T21:26:42,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742383_1559 (size=31161) 2024-11-17T21:26:42,451 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#C#compaction#472 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:42,452 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/cf5ed912578f4c22b23e016eb0f28eff is 50, key is test_row_0/C:col10/1731878799999/Put/seqid=0 2024-11-17T21:26:42,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742384_1560 (size=12207) 2024-11-17T21:26:42,463 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/cf5ed912578f4c22b23e016eb0f28eff as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/cf5ed912578f4c22b23e016eb0f28eff 2024-11-17T21:26:42,467 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/C of a21acdaa0aabfb075e55b063d1c8e884 into cf5ed912578f4c22b23e016eb0f28eff(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:42,467 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:42,467 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/C, priority=13, startTime=1731878802406; duration=0sec 2024-11-17T21:26:42,467 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:42,467 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:C 2024-11-17T21:26:42,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T21:26:42,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:42,513 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-17T21:26:42,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b4d78f7ca4db422a9b70c7915e8e3a30_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878801169/Put/seqid=0 2024-11-17T21:26:42,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742385_1561 (size=12154) 2024-11-17T21:26:42,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:42,526 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b4d78f7ca4db422a9b70c7915e8e3a30_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b4d78f7ca4db422a9b70c7915e8e3a30_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:42,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/cfe8aaa223c54cb2b55e50cd3a8db50b, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:42,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/cfe8aaa223c54cb2b55e50cd3a8db50b is 175, key is test_row_0/A:col10/1731878801169/Put/seqid=0 2024-11-17T21:26:42,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742386_1562 (size=30955) 2024-11-17T21:26:42,849 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/c484faa8dee04bf08eefb06f6f16c03d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/c484faa8dee04bf08eefb06f6f16c03d 2024-11-17T21:26:42,852 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/A of a21acdaa0aabfb075e55b063d1c8e884 into c484faa8dee04bf08eefb06f6f16c03d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:42,852 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:42,852 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/A, priority=13, startTime=1731878802406; duration=0sec 2024-11-17T21:26:42,853 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:42,853 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:A 2024-11-17T21:26:42,947 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/cfe8aaa223c54cb2b55e50cd3a8db50b 2024-11-17T21:26:42,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/3b40d34c79f34ab9b07c7c1b80a1c593 is 50, key is test_row_0/B:col10/1731878801169/Put/seqid=0 2024-11-17T21:26:42,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742387_1563 (size=12001) 2024-11-17T21:26:42,972 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/3b40d34c79f34ab9b07c7c1b80a1c593 2024-11-17T21:26:42,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/055173f5a2b5461e89294e18cf96e424 is 50, key is test_row_0/C:col10/1731878801169/Put/seqid=0 2024-11-17T21:26:42,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742388_1564 (size=12001) 2024-11-17T21:26:43,000 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/055173f5a2b5461e89294e18cf96e424 2024-11-17T21:26:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/cfe8aaa223c54cb2b55e50cd3a8db50b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/cfe8aaa223c54cb2b55e50cd3a8db50b 2024-11-17T21:26:43,009 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/cfe8aaa223c54cb2b55e50cd3a8db50b, entries=150, sequenceid=117, filesize=30.2 K 2024-11-17T21:26:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/3b40d34c79f34ab9b07c7c1b80a1c593 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/3b40d34c79f34ab9b07c7c1b80a1c593 2024-11-17T21:26:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,015 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/3b40d34c79f34ab9b07c7c1b80a1c593, entries=150, sequenceid=117, filesize=11.7 K 2024-11-17T21:26:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/055173f5a2b5461e89294e18cf96e424 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/055173f5a2b5461e89294e18cf96e424 2024-11-17T21:26:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,018 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/055173f5a2b5461e89294e18cf96e424, entries=150, sequenceid=117, filesize=11.7 K 2024-11-17T21:26:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,018 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for a21acdaa0aabfb075e55b063d1c8e884 in 505ms, sequenceid=117, compaction requested=false 2024-11-17T21:26:43,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:43,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:43,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-17T21:26:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-17T21:26:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-17T21:26:43,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7270 sec 2024-11-17T21:26:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.7310 sec 2024-11-17T21:26:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,368 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:43,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:26:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111719339669c89d468783f3b76d5a1c36b0_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878803367/Put/seqid=0 2024-11-17T21:26:43,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742389_1565 (size=26898) 2024-11-17T21:26:43,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878863440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:43,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878863441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:43,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878863441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T21:26:43,449 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:43,449 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-17T21:26:43,451 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T21:26:43,452 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:43,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:43,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878863551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:43,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:43,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878863551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:43,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:43,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878863551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:43,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T21:26:43,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T21:26:43,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:43,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:43,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:43,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:43,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:43,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T21:26:43,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:43,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:43,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:43,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:43,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:43,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:43,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878863761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878863761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:43,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878863762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T21:26:43,841 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:43,844 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111719339669c89d468783f3b76d5a1c36b0_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111719339669c89d468783f3b76d5a1c36b0_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:43,845 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/80f8b13bb2ab4bd79514cfbba05ff017, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:43,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/80f8b13bb2ab4bd79514cfbba05ff017 is 175, key is test_row_0/A:col10/1731878803367/Put/seqid=0 2024-11-17T21:26:43,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742390_1566 (size=82685) 2024-11-17T21:26:43,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T21:26:43,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:43,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:43,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:43,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:43,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:44,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T21:26:44,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:44,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:44,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:44,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:44,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:44,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878864071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:44,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:44,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:44,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878864071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:44,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:44,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:44,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:44,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:44,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878864071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:44,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:44,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:44,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T21:26:44,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T21:26:44,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:44,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:44,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:44,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:44,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:44,250 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/80f8b13bb2ab4bd79514cfbba05ff017 2024-11-17T21:26:44,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/aa700c464833434f87e400b407015a5b is 50, key is test_row_0/B:col10/1731878803367/Put/seqid=0 2024-11-17T21:26:44,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742391_1567 (size=12101) 2024-11-17T21:26:44,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T21:26:44,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:44,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:44,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:44,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:44,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:44,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T21:26:44,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:44,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:44,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878864581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:44,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878864581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T21:26:44,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878864590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:44,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/aa700c464833434f87e400b407015a5b 2024-11-17T21:26:44,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/330df5d5ec984cf3830795cfb01a14e2 is 50, key is test_row_0/C:col10/1731878803367/Put/seqid=0 2024-11-17T21:26:44,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T21:26:44,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:44,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:44,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:44,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:44,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:44,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742392_1568 (size=12101) 2024-11-17T21:26:44,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/330df5d5ec984cf3830795cfb01a14e2 2024-11-17T21:26:44,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/80f8b13bb2ab4bd79514cfbba05ff017 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/80f8b13bb2ab4bd79514cfbba05ff017 2024-11-17T21:26:44,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/80f8b13bb2ab4bd79514cfbba05ff017, entries=450, sequenceid=130, filesize=80.7 K 2024-11-17T21:26:44,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/aa700c464833434f87e400b407015a5b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/aa700c464833434f87e400b407015a5b 2024-11-17T21:26:44,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/aa700c464833434f87e400b407015a5b, entries=150, sequenceid=130, filesize=11.8 K 2024-11-17T21:26:44,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/330df5d5ec984cf3830795cfb01a14e2 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/330df5d5ec984cf3830795cfb01a14e2 2024-11-17T21:26:44,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/330df5d5ec984cf3830795cfb01a14e2, entries=150, sequenceid=130, filesize=11.8 K 2024-11-17T21:26:44,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for a21acdaa0aabfb075e55b063d1c8e884 in 1330ms, sequenceid=130, compaction requested=true 2024-11-17T21:26:44,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:44,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:26:44,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:44,698 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:44,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:26:44,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:44,698 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:44,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:26:44,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:44,699 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:44,699 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/B is initiating minor compaction (all files) 2024-11-17T21:26:44,699 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/B in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:44,700 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/4235733e698046fc83174f8e9db684d0, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/3b40d34c79f34ab9b07c7c1b80a1c593, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/aa700c464833434f87e400b407015a5b] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=35.5 K 2024-11-17T21:26:44,700 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144801 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:44,700 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/A is initiating minor compaction (all files) 2024-11-17T21:26:44,700 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/A in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:44,700 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/c484faa8dee04bf08eefb06f6f16c03d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/cfe8aaa223c54cb2b55e50cd3a8db50b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/80f8b13bb2ab4bd79514cfbba05ff017] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=141.4 K 2024-11-17T21:26:44,700 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:44,700 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/c484faa8dee04bf08eefb06f6f16c03d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/cfe8aaa223c54cb2b55e50cd3a8db50b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/80f8b13bb2ab4bd79514cfbba05ff017] 2024-11-17T21:26:44,700 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 4235733e698046fc83174f8e9db684d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731878799999 2024-11-17T21:26:44,700 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting c484faa8dee04bf08eefb06f6f16c03d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731878799999 2024-11-17T21:26:44,701 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b40d34c79f34ab9b07c7c1b80a1c593, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731878801148 2024-11-17T21:26:44,701 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfe8aaa223c54cb2b55e50cd3a8db50b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731878801148 2024-11-17T21:26:44,701 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting aa700c464833434f87e400b407015a5b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731878803364 2024-11-17T21:26:44,702 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80f8b13bb2ab4bd79514cfbba05ff017, keycount=450, bloomtype=ROW, size=80.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731878803350 2024-11-17T21:26:44,709 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:44,714 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#B#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:44,715 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/c22845a4c59f48d9a2f298ac3e0e50bf is 50, key is test_row_0/B:col10/1731878803367/Put/seqid=0 2024-11-17T21:26:44,718 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117a855fca7479c4544ba89def8bb07e2c3_a21acdaa0aabfb075e55b063d1c8e884 store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:44,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742393_1569 (size=12409) 2024-11-17T21:26:44,725 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117a855fca7479c4544ba89def8bb07e2c3_a21acdaa0aabfb075e55b063d1c8e884, store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:44,726 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a855fca7479c4544ba89def8bb07e2c3_a21acdaa0aabfb075e55b063d1c8e884 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:44,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742394_1570 (size=4469) 2024-11-17T21:26:44,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T21:26:44,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:44,829 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T21:26:44,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111739de58b4896b488e9ab8493e531860a4_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878803435/Put/seqid=0 2024-11-17T21:26:44,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742395_1571 (size=12304) 2024-11-17T21:26:44,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:44,845 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111739de58b4896b488e9ab8493e531860a4_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111739de58b4896b488e9ab8493e531860a4_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:44,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/2a62c88260764c468d86cca83f86a54d, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:44,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/2a62c88260764c468d86cca83f86a54d is 175, key is test_row_0/A:col10/1731878803435/Put/seqid=0 2024-11-17T21:26:44,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742396_1572 (size=31105) 2024-11-17T21:26:45,129 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/c22845a4c59f48d9a2f298ac3e0e50bf as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/c22845a4c59f48d9a2f298ac3e0e50bf 2024-11-17T21:26:45,134 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/B of a21acdaa0aabfb075e55b063d1c8e884 into c22845a4c59f48d9a2f298ac3e0e50bf(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:45,134 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:45,134 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/B, priority=13, startTime=1731878804698; duration=0sec 2024-11-17T21:26:45,134 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:45,134 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:B 2024-11-17T21:26:45,134 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:45,136 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:45,136 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/C is initiating minor compaction (all files) 2024-11-17T21:26:45,136 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/C in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:45,136 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/cf5ed912578f4c22b23e016eb0f28eff, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/055173f5a2b5461e89294e18cf96e424, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/330df5d5ec984cf3830795cfb01a14e2] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=35.5 K 2024-11-17T21:26:45,137 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting cf5ed912578f4c22b23e016eb0f28eff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731878799999 2024-11-17T21:26:45,137 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 055173f5a2b5461e89294e18cf96e424, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731878801148 2024-11-17T21:26:45,138 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 330df5d5ec984cf3830795cfb01a14e2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731878803364 2024-11-17T21:26:45,144 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#C#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:45,145 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/49b68ca1cc6042b1bc50ca96478b5823 is 50, key is test_row_0/C:col10/1731878803367/Put/seqid=0 2024-11-17T21:26:45,147 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#A#compaction#479 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:45,148 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/c5f24d7cc2b04ab8ad1d728b86a957b4 is 175, key is test_row_0/A:col10/1731878803367/Put/seqid=0 2024-11-17T21:26:45,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742398_1574 (size=31363) 2024-11-17T21:26:45,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742397_1573 (size=12409) 2024-11-17T21:26:45,175 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/49b68ca1cc6042b1bc50ca96478b5823 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/49b68ca1cc6042b1bc50ca96478b5823 2024-11-17T21:26:45,182 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/C of a21acdaa0aabfb075e55b063d1c8e884 into 49b68ca1cc6042b1bc50ca96478b5823(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:45,182 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:45,182 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/C, priority=13, startTime=1731878804698; duration=0sec 2024-11-17T21:26:45,182 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:45,182 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:C 2024-11-17T21:26:45,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:45,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:45,250 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/2a62c88260764c468d86cca83f86a54d 2024-11-17T21:26:45,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/a896f00f5eac40c6a350229d0d1e15b2 is 50, key is test_row_0/B:col10/1731878803435/Put/seqid=0 2024-11-17T21:26:45,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742399_1575 (size=12151) 2024-11-17T21:26:45,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878865274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:45,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52140 deadline: 1731878865281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:45,283 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_2' in a21acdaa0aabfb075e55b063d1c8e884 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:45,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878865380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:45,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:45,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:45,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:45,561 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/c5f24d7cc2b04ab8ad1d728b86a957b4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/c5f24d7cc2b04ab8ad1d728b86a957b4 2024-11-17T21:26:45,566 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/A of a21acdaa0aabfb075e55b063d1c8e884 into c5f24d7cc2b04ab8ad1d728b86a957b4(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:45,566 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:45,566 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/A, priority=13, startTime=1731878804698; duration=0sec 2024-11-17T21:26:45,566 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:45,566 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:A 2024-11-17T21:26:45,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878865591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:45,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878865591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:45,595 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in a21acdaa0aabfb075e55b063d1c8e884 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:45,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878865591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:45,595 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_0' in a21acdaa0aabfb075e55b063d1c8e884 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T21:26:45,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878865600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:45,604 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_0' in a21acdaa0aabfb075e55b063d1c8e884 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:45,666 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/a896f00f5eac40c6a350229d0d1e15b2 2024-11-17T21:26:45,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/4b3d3a4ab63f486cae4ce2d1c589270f is 50, key is test_row_0/C:col10/1731878803435/Put/seqid=0 2024-11-17T21:26:45,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742400_1576 (size=12151) 2024-11-17T21:26:45,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878865900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:45,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:45,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:45,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:46,082 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/4b3d3a4ab63f486cae4ce2d1c589270f 2024-11-17T21:26:46,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/2a62c88260764c468d86cca83f86a54d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/2a62c88260764c468d86cca83f86a54d 2024-11-17T21:26:46,088 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/2a62c88260764c468d86cca83f86a54d, entries=150, sequenceid=155, filesize=30.4 K 2024-11-17T21:26:46,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/a896f00f5eac40c6a350229d0d1e15b2 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/a896f00f5eac40c6a350229d0d1e15b2 2024-11-17T21:26:46,091 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/a896f00f5eac40c6a350229d0d1e15b2, entries=150, sequenceid=155, filesize=11.9 K 2024-11-17T21:26:46,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/4b3d3a4ab63f486cae4ce2d1c589270f as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/4b3d3a4ab63f486cae4ce2d1c589270f 2024-11-17T21:26:46,095 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/4b3d3a4ab63f486cae4ce2d1c589270f, entries=150, sequenceid=155, filesize=11.9 K 2024-11-17T21:26:46,096 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a21acdaa0aabfb075e55b063d1c8e884 in 1267ms, sequenceid=155, compaction requested=false 2024-11-17T21:26:46,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:46,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:46,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-17T21:26:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-17T21:26:46,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-17T21:26:46,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6460 sec 2024-11-17T21:26:46,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.6510 sec 2024-11-17T21:26:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:46,413 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:46,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T21:26:46,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117753384e957464af9bbb08cb74135abab_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878806412/Put/seqid=0 2024-11-17T21:26:46,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742401_1577 (size=14794) 2024-11-17T21:26:46,425 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:46,429 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117753384e957464af9bbb08cb74135abab_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117753384e957464af9bbb08cb74135abab_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:46,431 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/dc222d5043814b9f8626a32da3f130cd, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:46,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/dc222d5043814b9f8626a32da3f130cd is 175, key is test_row_0/A:col10/1731878806412/Put/seqid=0 2024-11-17T21:26:46,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742402_1578 (size=39749) 2024-11-17T21:26:46,439 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/dc222d5043814b9f8626a32da3f130cd 2024-11-17T21:26:46,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/646a84247df0445d910e64cfca05b0b4 is 50, key is test_row_0/B:col10/1731878806412/Put/seqid=0 2024-11-17T21:26:46,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742403_1579 (size=12151) 2024-11-17T21:26:46,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/646a84247df0445d910e64cfca05b0b4 2024-11-17T21:26:46,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/b7fb4e735d0248488c23bcbe75103d9a is 50, key is test_row_0/C:col10/1731878806412/Put/seqid=0 2024-11-17T21:26:46,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742404_1580 (size=12151) 2024-11-17T21:26:46,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878866524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:46,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878866630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:46,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:46,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:46,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:46,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878866841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:46,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:46,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:46,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:46,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/b7fb4e735d0248488c23bcbe75103d9a 2024-11-17T21:26:46,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/dc222d5043814b9f8626a32da3f130cd as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dc222d5043814b9f8626a32da3f130cd 2024-11-17T21:26:46,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dc222d5043814b9f8626a32da3f130cd, entries=200, sequenceid=170, filesize=38.8 K 2024-11-17T21:26:46,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/646a84247df0445d910e64cfca05b0b4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/646a84247df0445d910e64cfca05b0b4 2024-11-17T21:26:46,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/646a84247df0445d910e64cfca05b0b4, entries=150, sequenceid=170, filesize=11.9 K 2024-11-17T21:26:46,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/b7fb4e735d0248488c23bcbe75103d9a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/b7fb4e735d0248488c23bcbe75103d9a 2024-11-17T21:26:46,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/b7fb4e735d0248488c23bcbe75103d9a, entries=150, sequenceid=170, filesize=11.9 K 2024-11-17T21:26:46,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a21acdaa0aabfb075e55b063d1c8e884 in 494ms, sequenceid=170, compaction requested=true 2024-11-17T21:26:46,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:46,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:26:46,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:46,907 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:46,907 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:46,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:26:46,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:46,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:26:46,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:46,908 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:46,908 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:46,908 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/B is initiating minor compaction (all files) 2024-11-17T21:26:46,908 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/A is initiating minor compaction (all files) 2024-11-17T21:26:46,908 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/A in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:46,909 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/c5f24d7cc2b04ab8ad1d728b86a957b4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/2a62c88260764c468d86cca83f86a54d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dc222d5043814b9f8626a32da3f130cd] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=99.8 K 2024-11-17T21:26:46,909 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:46,909 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/c5f24d7cc2b04ab8ad1d728b86a957b4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/2a62c88260764c468d86cca83f86a54d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dc222d5043814b9f8626a32da3f130cd] 2024-11-17T21:26:46,909 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/B in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:46,909 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/c22845a4c59f48d9a2f298ac3e0e50bf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/a896f00f5eac40c6a350229d0d1e15b2, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/646a84247df0445d910e64cfca05b0b4] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=35.9 K 2024-11-17T21:26:46,910 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5f24d7cc2b04ab8ad1d728b86a957b4, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731878803364 2024-11-17T21:26:46,910 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting c22845a4c59f48d9a2f298ac3e0e50bf, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731878803364 2024-11-17T21:26:46,910 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a62c88260764c468d86cca83f86a54d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731878803425 2024-11-17T21:26:46,910 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting a896f00f5eac40c6a350229d0d1e15b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731878803425 2024-11-17T21:26:46,910 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc222d5043814b9f8626a32da3f130cd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731878805259 2024-11-17T21:26:46,910 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 646a84247df0445d910e64cfca05b0b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731878805259 2024-11-17T21:26:46,916 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#B#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:46,917 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/208ac44c26784fb38c170677bc98ce84 is 50, key is test_row_0/B:col10/1731878806412/Put/seqid=0 2024-11-17T21:26:46,918 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:46,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742405_1581 (size=12561) 2024-11-17T21:26:46,929 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117bf0db5e5c479461db31e1c28d4628d6d_a21acdaa0aabfb075e55b063d1c8e884 store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:46,930 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117bf0db5e5c479461db31e1c28d4628d6d_a21acdaa0aabfb075e55b063d1c8e884, store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:46,930 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bf0db5e5c479461db31e1c28d4628d6d_a21acdaa0aabfb075e55b063d1c8e884 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:46,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742406_1582 (size=4469) 2024-11-17T21:26:46,934 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#A#compaction#489 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:46,934 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/7f43b026e57a4dd197a46dc4181ea2ba is 175, key is test_row_0/A:col10/1731878806412/Put/seqid=0 2024-11-17T21:26:46,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742407_1583 (size=31515) 2024-11-17T21:26:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:47,154 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:47,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T21:26:47,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117804f4ba449af4267b4b572ebd7dffde6_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878806514/Put/seqid=0 2024-11-17T21:26:47,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742408_1584 (size=14794) 2024-11-17T21:26:47,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:47,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878867187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:47,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:47,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:47,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:47,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878867300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:47,329 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/208ac44c26784fb38c170677bc98ce84 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/208ac44c26784fb38c170677bc98ce84 2024-11-17T21:26:47,333 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/B of a21acdaa0aabfb075e55b063d1c8e884 into 208ac44c26784fb38c170677bc98ce84(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:47,333 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:47,333 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/B, priority=13, startTime=1731878806907; duration=0sec 2024-11-17T21:26:47,333 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:47,333 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:B 2024-11-17T21:26:47,333 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:47,334 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:47,334 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/C is initiating minor compaction (all files) 2024-11-17T21:26:47,334 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/C in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:47,334 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/49b68ca1cc6042b1bc50ca96478b5823, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/4b3d3a4ab63f486cae4ce2d1c589270f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/b7fb4e735d0248488c23bcbe75103d9a] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=35.9 K 2024-11-17T21:26:47,334 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 49b68ca1cc6042b1bc50ca96478b5823, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731878803364 2024-11-17T21:26:47,334 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b3d3a4ab63f486cae4ce2d1c589270f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731878803425 2024-11-17T21:26:47,335 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting b7fb4e735d0248488c23bcbe75103d9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731878805259 2024-11-17T21:26:47,341 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/7f43b026e57a4dd197a46dc4181ea2ba as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/7f43b026e57a4dd197a46dc4181ea2ba 2024-11-17T21:26:47,341 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#C#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:47,342 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/9ae4cde614f54cd6a8711df93894d556 is 50, key is test_row_0/C:col10/1731878806412/Put/seqid=0 2024-11-17T21:26:47,345 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/A of a21acdaa0aabfb075e55b063d1c8e884 into 7f43b026e57a4dd197a46dc4181ea2ba(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:47,345 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:47,345 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/A, priority=13, startTime=1731878806907; duration=0sec 2024-11-17T21:26:47,345 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:47,346 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:A 2024-11-17T21:26:47,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742409_1585 (size=12561) 2024-11-17T21:26:47,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878867511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:47,589 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:47,592 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117804f4ba449af4267b4b572ebd7dffde6_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117804f4ba449af4267b4b572ebd7dffde6_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:47,593 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/1a487c44355e49a19c6b3242c3ca080d, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:47,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/1a487c44355e49a19c6b3242c3ca080d is 175, key is test_row_0/A:col10/1731878806514/Put/seqid=0 2024-11-17T21:26:47,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742410_1586 (size=39749) 2024-11-17T21:26:47,596 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/1a487c44355e49a19c6b3242c3ca080d 2024-11-17T21:26:47,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/9a6ef4a4dd3d4f21ab3111a6ffced507 is 50, key is test_row_0/B:col10/1731878806514/Put/seqid=0 2024-11-17T21:26:47,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742411_1587 (size=12151) 2024-11-17T21:26:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T21:26:47,621 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:47,622 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:47,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-17T21:26:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878867621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:47,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878867621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:47,623 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_1' in a21acdaa0aabfb075e55b063d1c8e884 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:47,623 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_0' in a21acdaa0aabfb075e55b063d1c8e884 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T21:26:47,624 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:47,625 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:47,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:47,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878867631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:47,635 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_0' in a21acdaa0aabfb075e55b063d1c8e884 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T21:26:47,757 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/9ae4cde614f54cd6a8711df93894d556 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/9ae4cde614f54cd6a8711df93894d556 2024-11-17T21:26:47,768 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/C of a21acdaa0aabfb075e55b063d1c8e884 into 9ae4cde614f54cd6a8711df93894d556(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:47,768 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:47,768 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/C, priority=13, startTime=1731878806908; duration=0sec 2024-11-17T21:26:47,768 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:47,768 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:C 2024-11-17T21:26:47,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T21:26:47,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:47,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:47,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:47,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:47,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:47,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878867821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:47,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T21:26:47,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:47,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:47,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:47,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:47,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T21:26:48,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/9a6ef4a4dd3d4f21ab3111a6ffced507 2024-11-17T21:26:48,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/18dbd3ae869040b2ba60898f63798c96 is 50, key is test_row_0/C:col10/1731878806514/Put/seqid=0 2024-11-17T21:26:48,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742412_1588 (size=12151) 2024-11-17T21:26:48,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T21:26:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:48,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:48,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T21:26:48,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:48,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:48,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:48,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:48,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T21:26:48,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878868331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:48,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T21:26:48,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:48,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:48,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:48,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:48,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:48,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/18dbd3ae869040b2ba60898f63798c96 2024-11-17T21:26:48,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/1a487c44355e49a19c6b3242c3ca080d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/1a487c44355e49a19c6b3242c3ca080d 2024-11-17T21:26:48,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/1a487c44355e49a19c6b3242c3ca080d, entries=200, sequenceid=195, filesize=38.8 K 2024-11-17T21:26:48,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/9a6ef4a4dd3d4f21ab3111a6ffced507 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/9a6ef4a4dd3d4f21ab3111a6ffced507 2024-11-17T21:26:48,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/9a6ef4a4dd3d4f21ab3111a6ffced507, entries=150, sequenceid=195, filesize=11.9 K 2024-11-17T21:26:48,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/18dbd3ae869040b2ba60898f63798c96 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/18dbd3ae869040b2ba60898f63798c96 2024-11-17T21:26:48,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/18dbd3ae869040b2ba60898f63798c96, entries=150, sequenceid=195, filesize=11.9 K 2024-11-17T21:26:48,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a21acdaa0aabfb075e55b063d1c8e884 in 1288ms, sequenceid=195, compaction requested=false 2024-11-17T21:26:48,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:48,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T21:26:48,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:48,541 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:26:48,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f32d230e39884c88a3b5fa727b70805f_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878807181/Put/seqid=0 2024-11-17T21:26:48,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742413_1589 (size=12304) 2024-11-17T21:26:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T21:26:48,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:48,980 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f32d230e39884c88a3b5fa727b70805f_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f32d230e39884c88a3b5fa727b70805f_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:48,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/2e02c5d7786a4b05b0beace293888850, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:48,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/2e02c5d7786a4b05b0beace293888850 is 175, key is test_row_0/A:col10/1731878807181/Put/seqid=0 2024-11-17T21:26:48,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742414_1590 (size=31105) 2024-11-17T21:26:49,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:49,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:49,386 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/2e02c5d7786a4b05b0beace293888850 2024-11-17T21:26:49,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/1e77b6b570b84cd1bb8a8f2fab296ad9 is 50, key is test_row_0/B:col10/1731878807181/Put/seqid=0 2024-11-17T21:26:49,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742415_1591 (size=12151) 2024-11-17T21:26:49,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878869427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:49,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878869540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:49,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878869751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T21:26:49,806 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/1e77b6b570b84cd1bb8a8f2fab296ad9 2024-11-17T21:26:49,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/a7187495fb434b75a130898c041fe291 is 50, key is test_row_0/C:col10/1731878807181/Put/seqid=0 2024-11-17T21:26:49,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742416_1592 (size=12151) 2024-11-17T21:26:50,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878870060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:50,216 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/a7187495fb434b75a130898c041fe291 2024-11-17T21:26:50,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/2e02c5d7786a4b05b0beace293888850 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/2e02c5d7786a4b05b0beace293888850 2024-11-17T21:26:50,222 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/2e02c5d7786a4b05b0beace293888850, entries=150, sequenceid=209, filesize=30.4 K 2024-11-17T21:26:50,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/1e77b6b570b84cd1bb8a8f2fab296ad9 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/1e77b6b570b84cd1bb8a8f2fab296ad9 2024-11-17T21:26:50,227 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/1e77b6b570b84cd1bb8a8f2fab296ad9, entries=150, sequenceid=209, filesize=11.9 K 2024-11-17T21:26:50,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/a7187495fb434b75a130898c041fe291 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/a7187495fb434b75a130898c041fe291 2024-11-17T21:26:50,230 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/a7187495fb434b75a130898c041fe291, entries=150, sequenceid=209, filesize=11.9 K 2024-11-17T21:26:50,231 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a21acdaa0aabfb075e55b063d1c8e884 in 1690ms, sequenceid=209, compaction requested=true 2024-11-17T21:26:50,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:50,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:50,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-17T21:26:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-17T21:26:50,233 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-17T21:26:50,233 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6070 sec 2024-11-17T21:26:50,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.6120 sec 2024-11-17T21:26:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:50,573 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:50,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T21:26:50,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171356ca3c8ea9437891d462517db2dd56_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878810572/Put/seqid=0 2024-11-17T21:26:50,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742417_1593 (size=14794) 2024-11-17T21:26:50,590 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:50,595 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171356ca3c8ea9437891d462517db2dd56_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171356ca3c8ea9437891d462517db2dd56_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:50,596 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/a6f98bc3cdfe483eaa8144d5a52787bc, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:50,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/a6f98bc3cdfe483eaa8144d5a52787bc is 175, key is test_row_0/A:col10/1731878810572/Put/seqid=0 2024-11-17T21:26:50,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742418_1594 (size=39749) 2024-11-17T21:26:50,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878870606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:50,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:50,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:50,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:50,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878870721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:50,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878870931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:51,000 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/a6f98bc3cdfe483eaa8144d5a52787bc 2024-11-17T21:26:51,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/e0a087501bd3465bbefb68e3e9ae5896 is 50, key is test_row_0/B:col10/1731878810572/Put/seqid=0 2024-11-17T21:26:51,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742419_1595 (size=12151) 2024-11-17T21:26:51,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/e0a087501bd3465bbefb68e3e9ae5896 2024-11-17T21:26:51,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/3b4e6454c5f44936836818ec2f562b30 is 50, key is test_row_0/C:col10/1731878810572/Put/seqid=0 2024-11-17T21:26:51,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742420_1596 (size=12151) 2024-11-17T21:26:51,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878871240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:51,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/3b4e6454c5f44936836818ec2f562b30 2024-11-17T21:26:51,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/a6f98bc3cdfe483eaa8144d5a52787bc as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/a6f98bc3cdfe483eaa8144d5a52787bc 2024-11-17T21:26:51,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/a6f98bc3cdfe483eaa8144d5a52787bc, entries=200, sequenceid=235, filesize=38.8 K 2024-11-17T21:26:51,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/e0a087501bd3465bbefb68e3e9ae5896 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/e0a087501bd3465bbefb68e3e9ae5896 2024-11-17T21:26:51,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/e0a087501bd3465bbefb68e3e9ae5896, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T21:26:51,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/3b4e6454c5f44936836818ec2f562b30 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/3b4e6454c5f44936836818ec2f562b30 2024-11-17T21:26:51,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/3b4e6454c5f44936836818ec2f562b30, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T21:26:51,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for a21acdaa0aabfb075e55b063d1c8e884 in 863ms, sequenceid=235, compaction requested=true 2024-11-17T21:26:51,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:51,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:26:51,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:51,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:26:51,437 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:26:51,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:51,437 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:26:51,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:26:51,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:51,440 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:26:51,441 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/B is initiating minor compaction (all files) 2024-11-17T21:26:51,441 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/B in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:51,441 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/208ac44c26784fb38c170677bc98ce84, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/9a6ef4a4dd3d4f21ab3111a6ffced507, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/1e77b6b570b84cd1bb8a8f2fab296ad9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/e0a087501bd3465bbefb68e3e9ae5896] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=47.9 K 2024-11-17T21:26:51,441 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142118 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:26:51,441 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/A is initiating minor compaction (all files) 2024-11-17T21:26:51,441 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/A in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:51,441 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 208ac44c26784fb38c170677bc98ce84, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731878805259 2024-11-17T21:26:51,441 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/7f43b026e57a4dd197a46dc4181ea2ba, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/1a487c44355e49a19c6b3242c3ca080d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/2e02c5d7786a4b05b0beace293888850, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/a6f98bc3cdfe483eaa8144d5a52787bc] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=138.8 K 2024-11-17T21:26:51,441 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:51,441 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/7f43b026e57a4dd197a46dc4181ea2ba, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/1a487c44355e49a19c6b3242c3ca080d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/2e02c5d7786a4b05b0beace293888850, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/a6f98bc3cdfe483eaa8144d5a52787bc] 2024-11-17T21:26:51,442 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f43b026e57a4dd197a46dc4181ea2ba, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731878805259 2024-11-17T21:26:51,442 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a6ef4a4dd3d4f21ab3111a6ffced507, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731878806509 2024-11-17T21:26:51,442 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e77b6b570b84cd1bb8a8f2fab296ad9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731878807169 2024-11-17T21:26:51,442 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a487c44355e49a19c6b3242c3ca080d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731878806509 2024-11-17T21:26:51,443 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting e0a087501bd3465bbefb68e3e9ae5896, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731878809418 2024-11-17T21:26:51,443 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e02c5d7786a4b05b0beace293888850, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731878807169 2024-11-17T21:26:51,443 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6f98bc3cdfe483eaa8144d5a52787bc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731878809418 2024-11-17T21:26:51,450 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:51,452 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#B#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:51,453 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111770651bf4e32c4befaf28750add09d43c_a21acdaa0aabfb075e55b063d1c8e884 store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:51,453 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/f053dea396f24438a31faf53aa86ba53 is 50, key is test_row_0/B:col10/1731878810572/Put/seqid=0 2024-11-17T21:26:51,455 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111770651bf4e32c4befaf28750add09d43c_a21acdaa0aabfb075e55b063d1c8e884, store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:51,456 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111770651bf4e32c4befaf28750add09d43c_a21acdaa0aabfb075e55b063d1c8e884 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:51,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742421_1597 (size=12697) 2024-11-17T21:26:51,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742422_1598 (size=4469) 2024-11-17T21:26:51,462 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#A#compaction#500 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:51,463 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/7d5dbfd9a5e945b3853f128f4e58dc3a is 175, key is test_row_0/A:col10/1731878810572/Put/seqid=0 2024-11-17T21:26:51,466 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/f053dea396f24438a31faf53aa86ba53 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/f053dea396f24438a31faf53aa86ba53 2024-11-17T21:26:51,471 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/B of a21acdaa0aabfb075e55b063d1c8e884 into f053dea396f24438a31faf53aa86ba53(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:51,471 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:51,471 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/B, priority=12, startTime=1731878811437; duration=0sec 2024-11-17T21:26:51,471 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:51,471 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:B 2024-11-17T21:26:51,471 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:26:51,472 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:26:51,472 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/C is initiating minor compaction (all files) 2024-11-17T21:26:51,472 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/C in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:51,472 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/9ae4cde614f54cd6a8711df93894d556, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/18dbd3ae869040b2ba60898f63798c96, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/a7187495fb434b75a130898c041fe291, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/3b4e6454c5f44936836818ec2f562b30] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=47.9 K 2024-11-17T21:26:51,472 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ae4cde614f54cd6a8711df93894d556, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731878805259 2024-11-17T21:26:51,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742423_1599 (size=31651) 2024-11-17T21:26:51,473 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 18dbd3ae869040b2ba60898f63798c96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731878806509 2024-11-17T21:26:51,473 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting a7187495fb434b75a130898c041fe291, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731878807169 2024-11-17T21:26:51,474 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b4e6454c5f44936836818ec2f562b30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731878809418 2024-11-17T21:26:51,494 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#C#compaction#502 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:51,494 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/26ab53aa9df5479fa05d37d8ee6fd49a is 50, key is test_row_0/C:col10/1731878810572/Put/seqid=0 2024-11-17T21:26:51,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742424_1600 (size=12697) 2024-11-17T21:26:51,508 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/26ab53aa9df5479fa05d37d8ee6fd49a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/26ab53aa9df5479fa05d37d8ee6fd49a 2024-11-17T21:26:51,513 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/C of a21acdaa0aabfb075e55b063d1c8e884 into 26ab53aa9df5479fa05d37d8ee6fd49a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:51,513 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:51,513 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/C, priority=12, startTime=1731878811437; duration=0sec 2024-11-17T21:26:51,513 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:51,513 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:C 2024-11-17T21:26:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:51,642 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:51,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:26:51,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111786d9b47086d34f1488e19064e462f628_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878811642/Put/seqid=0 2024-11-17T21:26:51,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742425_1601 (size=12304) 2024-11-17T21:26:51,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878871678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:51,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:51,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:51,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:51,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878871680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:51,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:51,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:51,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:51,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878871683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:51,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:51,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:51,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:51,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878871751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:51,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:51,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:51,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:51,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878871790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:51,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878871791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:51,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878871791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T21:26:51,811 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:51,811 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-17T21:26:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T21:26:51,813 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:51,814 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:51,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:51,877 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/7d5dbfd9a5e945b3853f128f4e58dc3a as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/7d5dbfd9a5e945b3853f128f4e58dc3a 2024-11-17T21:26:51,881 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/A of a21acdaa0aabfb075e55b063d1c8e884 into 7d5dbfd9a5e945b3853f128f4e58dc3a(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:51,881 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:51,881 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/A, priority=12, startTime=1731878811437; duration=0sec 2024-11-17T21:26:51,881 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:51,881 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:A 2024-11-17T21:26:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T21:26:51,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T21:26:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:51,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878872001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:52,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:52,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878872001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:52,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878872001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:52,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:52,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:52,070 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:52,072 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111786d9b47086d34f1488e19064e462f628_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111786d9b47086d34f1488e19064e462f628_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:52,073 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/52a72647b37941a8af45fd3541f1129e, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:52,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/52a72647b37941a8af45fd3541f1129e is 175, key is test_row_0/A:col10/1731878811642/Put/seqid=0 2024-11-17T21:26:52,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742426_1602 (size=31105) 2024-11-17T21:26:52,089 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/52a72647b37941a8af45fd3541f1129e 2024-11-17T21:26:52,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/d122d18498e745d1abf0ef239c9c8017 is 50, key is test_row_0/B:col10/1731878811642/Put/seqid=0 2024-11-17T21:26:52,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742427_1603 (size=12151) 2024-11-17T21:26:52,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T21:26:52,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:52,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:52,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:52,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T21:26:52,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T21:26:52,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:52,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:52,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:52,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878872311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:52,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878872311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:52,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878872311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:52,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T21:26:52,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:52,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:52,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:52,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T21:26:52,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/d122d18498e745d1abf0ef239c9c8017 2024-11-17T21:26:52,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/1a61996b615e4e8bb27b5a3db099bc7f is 50, key is test_row_0/C:col10/1731878811642/Put/seqid=0 2024-11-17T21:26:52,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742428_1604 (size=12151) 2024-11-17T21:26:52,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T21:26:52,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:52,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:52,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:52,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T21:26:52,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:52,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:52,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:52,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878872771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:52,772 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_0' in a21acdaa0aabfb075e55b063d1c8e884 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:52,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878872821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878872821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878872821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:52,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T21:26:52,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:52,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:52,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:52,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:52,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/1a61996b615e4e8bb27b5a3db099bc7f 2024-11-17T21:26:52,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/52a72647b37941a8af45fd3541f1129e as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/52a72647b37941a8af45fd3541f1129e 2024-11-17T21:26:52,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/52a72647b37941a8af45fd3541f1129e, entries=150, sequenceid=248, filesize=30.4 K 2024-11-17T21:26:52,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/d122d18498e745d1abf0ef239c9c8017 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/d122d18498e745d1abf0ef239c9c8017 2024-11-17T21:26:52,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/d122d18498e745d1abf0ef239c9c8017, entries=150, sequenceid=248, filesize=11.9 K 2024-11-17T21:26:52,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/1a61996b615e4e8bb27b5a3db099bc7f as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/1a61996b615e4e8bb27b5a3db099bc7f 2024-11-17T21:26:52,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/1a61996b615e4e8bb27b5a3db099bc7f, entries=150, sequenceid=248, filesize=11.9 K 2024-11-17T21:26:52,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a21acdaa0aabfb075e55b063d1c8e884 in 1294ms, sequenceid=248, compaction requested=false 2024-11-17T21:26:52,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T21:26:53,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T21:26:53,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:53,034 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T21:26:53,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178f319ce591a04264a73d7a8574749c48_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878811679/Put/seqid=0 2024-11-17T21:26:53,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742429_1605 (size=12454) 2024-11-17T21:26:53,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:53,054 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178f319ce591a04264a73d7a8574749c48_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178f319ce591a04264a73d7a8574749c48_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:53,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/9b662bc1ca1d48fc8bdc9a11b80d1494, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:53,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/9b662bc1ca1d48fc8bdc9a11b80d1494 is 175, key is test_row_0/A:col10/1731878811679/Put/seqid=0 2024-11-17T21:26:53,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742430_1606 (size=31255) 2024-11-17T21:26:53,472 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/9b662bc1ca1d48fc8bdc9a11b80d1494 2024-11-17T21:26:53,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/5ba7bdf9bb4a484c83c839d6eb44a518 is 50, key is test_row_0/B:col10/1731878811679/Put/seqid=0 2024-11-17T21:26:53,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742431_1607 (size=12301) 2024-11-17T21:26:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:53,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:53,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878873847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:53,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878873848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:53,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:53,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:53,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:53,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878873849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:53,886 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/5ba7bdf9bb4a484c83c839d6eb44a518 2024-11-17T21:26:53,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/d6e794dc80724b72ad536a2e5e69c2e4 is 50, key is test_row_0/C:col10/1731878811679/Put/seqid=0 2024-11-17T21:26:53,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742432_1608 (size=12301) 2024-11-17T21:26:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T21:26:53,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878873961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:53,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878873961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:53,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878873961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:54,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878874171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:54,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:54,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878874171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878874171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:54,294 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/d6e794dc80724b72ad536a2e5e69c2e4 2024-11-17T21:26:54,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/9b662bc1ca1d48fc8bdc9a11b80d1494 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/9b662bc1ca1d48fc8bdc9a11b80d1494 2024-11-17T21:26:54,300 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/9b662bc1ca1d48fc8bdc9a11b80d1494, entries=150, sequenceid=274, filesize=30.5 K 2024-11-17T21:26:54,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/5ba7bdf9bb4a484c83c839d6eb44a518 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/5ba7bdf9bb4a484c83c839d6eb44a518 2024-11-17T21:26:54,311 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/5ba7bdf9bb4a484c83c839d6eb44a518, entries=150, sequenceid=274, filesize=12.0 K 2024-11-17T21:26:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/d6e794dc80724b72ad536a2e5e69c2e4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/d6e794dc80724b72ad536a2e5e69c2e4 2024-11-17T21:26:54,314 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/d6e794dc80724b72ad536a2e5e69c2e4, entries=150, sequenceid=274, filesize=12.0 K 2024-11-17T21:26:54,315 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a21acdaa0aabfb075e55b063d1c8e884 in 1281ms, sequenceid=274, compaction requested=true 2024-11-17T21:26:54,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:54,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:54,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-17T21:26:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-17T21:26:54,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-17T21:26:54,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5020 sec 2024-11-17T21:26:54,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.5070 sec 2024-11-17T21:26:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:54,485 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:54,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T21:26:54,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179e51dcba7fd64d69a551034f4453f7df_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878813848/Put/seqid=0 2024-11-17T21:26:54,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878874514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:54,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:54,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742433_1609 (size=17534) 2024-11-17T21:26:54,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878874518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:54,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878874519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:54,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878874631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:54,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878874631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878874631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:54,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52172 deadline: 1731878874801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:54,804 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_0' in a21acdaa0aabfb075e55b063d1c8e884 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:54,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878874841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:54,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:54,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878874841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:54,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:54,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:54,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878874841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:54,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:54,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:54,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:54,922 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:54,925 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179e51dcba7fd64d69a551034f4453f7df_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179e51dcba7fd64d69a551034f4453f7df_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:54,926 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/15aeb7c2c5c94c1d9d2f095b8cb412ae, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:54,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/15aeb7c2c5c94c1d9d2f095b8cb412ae is 175, key is test_row_0/A:col10/1731878813848/Put/seqid=0 2024-11-17T21:26:54,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742434_1610 (size=48639) 2024-11-17T21:26:54,929 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/15aeb7c2c5c94c1d9d2f095b8cb412ae 2024-11-17T21:26:54,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/8d3a8a007cb141d5b191bfb23907d207 is 50, key is test_row_0/B:col10/1731878813848/Put/seqid=0 2024-11-17T21:26:54,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742435_1611 (size=12301) 2024-11-17T21:26:55,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878875150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:55,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878875151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:55,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878875151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:55,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/8d3a8a007cb141d5b191bfb23907d207 2024-11-17T21:26:55,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/2da40d962f2e48a0b36c26828924c99f is 50, key is test_row_0/C:col10/1731878813848/Put/seqid=0 2024-11-17T21:26:55,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52140 deadline: 1731878875341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:55,343 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a313eea8709e,46445,1731878642828 for 'test_row_2' in a21acdaa0aabfb075e55b063d1c8e884 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18305 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T21:26:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:55,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742436_1612 (size=12301) 2024-11-17T21:26:55,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878875660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:55,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:55,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:55,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878875661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:55,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878875661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:55,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/2da40d962f2e48a0b36c26828924c99f 2024-11-17T21:26:55,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/15aeb7c2c5c94c1d9d2f095b8cb412ae as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/15aeb7c2c5c94c1d9d2f095b8cb412ae 2024-11-17T21:26:55,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/15aeb7c2c5c94c1d9d2f095b8cb412ae, entries=250, sequenceid=288, filesize=47.5 K 2024-11-17T21:26:55,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/8d3a8a007cb141d5b191bfb23907d207 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/8d3a8a007cb141d5b191bfb23907d207 2024-11-17T21:26:55,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/8d3a8a007cb141d5b191bfb23907d207, entries=150, sequenceid=288, filesize=12.0 K 2024-11-17T21:26:55,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/2da40d962f2e48a0b36c26828924c99f as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/2da40d962f2e48a0b36c26828924c99f 2024-11-17T21:26:55,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/2da40d962f2e48a0b36c26828924c99f, entries=150, sequenceid=288, filesize=12.0 K 2024-11-17T21:26:55,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a21acdaa0aabfb075e55b063d1c8e884 in 1279ms, sequenceid=288, compaction requested=true 2024-11-17T21:26:55,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:55,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:26:55,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:55,764 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:26:55,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:26:55,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:55,764 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:26:55,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:26:55,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:55,765 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142650 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:26:55,765 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:26:55,765 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/B is initiating minor compaction (all files) 2024-11-17T21:26:55,765 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/A is initiating minor compaction (all files) 2024-11-17T21:26:55,765 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/A in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:55,765 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/B in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:55,765 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/7d5dbfd9a5e945b3853f128f4e58dc3a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/52a72647b37941a8af45fd3541f1129e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/9b662bc1ca1d48fc8bdc9a11b80d1494, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/15aeb7c2c5c94c1d9d2f095b8cb412ae] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=139.3 K 2024-11-17T21:26:55,765 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/f053dea396f24438a31faf53aa86ba53, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/d122d18498e745d1abf0ef239c9c8017, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/5ba7bdf9bb4a484c83c839d6eb44a518, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/8d3a8a007cb141d5b191bfb23907d207] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=48.3 K 2024-11-17T21:26:55,765 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:55,765 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/7d5dbfd9a5e945b3853f128f4e58dc3a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/52a72647b37941a8af45fd3541f1129e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/9b662bc1ca1d48fc8bdc9a11b80d1494, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/15aeb7c2c5c94c1d9d2f095b8cb412ae] 2024-11-17T21:26:55,765 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting f053dea396f24438a31faf53aa86ba53, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731878809418 2024-11-17T21:26:55,766 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d5dbfd9a5e945b3853f128f4e58dc3a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731878809418 2024-11-17T21:26:55,766 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting d122d18498e745d1abf0ef239c9c8017, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731878810589 2024-11-17T21:26:55,766 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52a72647b37941a8af45fd3541f1129e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731878810589 2024-11-17T21:26:55,766 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ba7bdf9bb4a484c83c839d6eb44a518, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731878811670 2024-11-17T21:26:55,766 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b662bc1ca1d48fc8bdc9a11b80d1494, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731878811670 2024-11-17T21:26:55,766 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d3a8a007cb141d5b191bfb23907d207, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731878813843 2024-11-17T21:26:55,766 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15aeb7c2c5c94c1d9d2f095b8cb412ae, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731878813843 2024-11-17T21:26:55,774 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#B#compaction#512 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:55,774 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/f5c497a811384d29816b6c133a6b25d7 is 50, key is test_row_0/B:col10/1731878813848/Put/seqid=0 2024-11-17T21:26:55,775 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:55,785 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411173ea69a69edf7475b8eab66b9259b7533_a21acdaa0aabfb075e55b063d1c8e884 store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:55,788 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411173ea69a69edf7475b8eab66b9259b7533_a21acdaa0aabfb075e55b063d1c8e884, store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:55,788 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173ea69a69edf7475b8eab66b9259b7533_a21acdaa0aabfb075e55b063d1c8e884 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:55,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742437_1613 (size=12983) 2024-11-17T21:26:55,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742438_1614 (size=4469) 2024-11-17T21:26:55,797 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#A#compaction#513 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:55,798 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/379ab3072efa4a9680c87085318282c7 is 175, key is test_row_0/A:col10/1731878813848/Put/seqid=0 2024-11-17T21:26:55,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742439_1615 (size=31937) 2024-11-17T21:26:55,814 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/379ab3072efa4a9680c87085318282c7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/379ab3072efa4a9680c87085318282c7 2024-11-17T21:26:55,819 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/A of a21acdaa0aabfb075e55b063d1c8e884 into 379ab3072efa4a9680c87085318282c7(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:55,819 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:55,819 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/A, priority=12, startTime=1731878815764; duration=0sec 2024-11-17T21:26:55,819 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:55,819 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:A 2024-11-17T21:26:55,819 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T21:26:55,820 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T21:26:55,820 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/C is initiating minor compaction (all files) 2024-11-17T21:26:55,820 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/C in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:55,820 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/26ab53aa9df5479fa05d37d8ee6fd49a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/1a61996b615e4e8bb27b5a3db099bc7f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/d6e794dc80724b72ad536a2e5e69c2e4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/2da40d962f2e48a0b36c26828924c99f] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=48.3 K 2024-11-17T21:26:55,821 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26ab53aa9df5479fa05d37d8ee6fd49a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731878809418 2024-11-17T21:26:55,821 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a61996b615e4e8bb27b5a3db099bc7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731878810589 2024-11-17T21:26:55,822 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6e794dc80724b72ad536a2e5e69c2e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731878811670 2024-11-17T21:26:55,822 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2da40d962f2e48a0b36c26828924c99f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731878813843 2024-11-17T21:26:55,829 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#C#compaction#514 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:55,830 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/0ac9a230b8b64c86a1ea32588c59d756 is 50, key is test_row_0/C:col10/1731878813848/Put/seqid=0 2024-11-17T21:26:55,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742440_1616 (size=12983) 2024-11-17T21:26:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T21:26:55,981 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:55,982 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-17T21:26:55,983 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T21:26:55,984 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:55,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T21:26:56,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T21:26:56,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:56,136 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T21:26:56,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111783e98509146b42a2a96a1542c1976cbb_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878814517/Put/seqid=0 2024-11-17T21:26:56,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742441_1617 (size=12454) 2024-11-17T21:26:56,194 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/f5c497a811384d29816b6c133a6b25d7 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/f5c497a811384d29816b6c133a6b25d7 2024-11-17T21:26:56,198 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/B of a21acdaa0aabfb075e55b063d1c8e884 into f5c497a811384d29816b6c133a6b25d7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:56,198 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:56,198 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/B, priority=12, startTime=1731878815764; duration=0sec 2024-11-17T21:26:56,198 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:56,198 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:B 2024-11-17T21:26:56,245 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/0ac9a230b8b64c86a1ea32588c59d756 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/0ac9a230b8b64c86a1ea32588c59d756 2024-11-17T21:26:56,250 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/C of a21acdaa0aabfb075e55b063d1c8e884 into 0ac9a230b8b64c86a1ea32588c59d756(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:56,250 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:56,250 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/C, priority=12, startTime=1731878815764; duration=0sec 2024-11-17T21:26:56,250 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:56,250 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:C 2024-11-17T21:26:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T21:26:56,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,549 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111783e98509146b42a2a96a1542c1976cbb_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111783e98509146b42a2a96a1542c1976cbb_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:56,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/9e42292fa7fd43adafce78b03be762a9, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:56,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/9e42292fa7fd43adafce78b03be762a9 is 175, key is test_row_0/A:col10/1731878814517/Put/seqid=0 2024-11-17T21:26:56,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742442_1618 (size=31255) 2024-11-17T21:26:56,554 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/9e42292fa7fd43adafce78b03be762a9 2024-11-17T21:26:56,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/6f3d3387ad3e478fb1b55f9314074b96 is 50, key is test_row_0/B:col10/1731878814517/Put/seqid=0 2024-11-17T21:26:56,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742443_1619 (size=12301) 2024-11-17T21:26:56,566 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/6f3d3387ad3e478fb1b55f9314074b96 2024-11-17T21:26:56,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/a00941d9d87f44d1888191d92505f171 is 50, key is test_row_0/C:col10/1731878814517/Put/seqid=0 2024-11-17T21:26:56,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742444_1620 (size=12301) 2024-11-17T21:26:56,576 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/a00941d9d87f44d1888191d92505f171 2024-11-17T21:26:56,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/9e42292fa7fd43adafce78b03be762a9 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/9e42292fa7fd43adafce78b03be762a9 2024-11-17T21:26:56,584 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/9e42292fa7fd43adafce78b03be762a9, entries=150, sequenceid=311, filesize=30.5 K 2024-11-17T21:26:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/6f3d3387ad3e478fb1b55f9314074b96 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/6f3d3387ad3e478fb1b55f9314074b96 2024-11-17T21:26:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,594 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/6f3d3387ad3e478fb1b55f9314074b96, entries=150, sequenceid=311, filesize=12.0 K 2024-11-17T21:26:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/a00941d9d87f44d1888191d92505f171 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/a00941d9d87f44d1888191d92505f171 2024-11-17T21:26:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,599 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/a00941d9d87f44d1888191d92505f171, entries=150, sequenceid=311, filesize=12.0 K 2024-11-17T21:26:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,600 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for a21acdaa0aabfb075e55b063d1c8e884 in 464ms, sequenceid=311, compaction requested=false 2024-11-17T21:26:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:56,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-17T21:26:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-17T21:26:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-17T21:26:56,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 618 msec 2024-11-17T21:26:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 623 msec 2024-11-17T21:26:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T21:26:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,611 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:26:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,612 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T21:26:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-17T21:26:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T21:26:56,614 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T21:26:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,614 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T21:26:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T21:26:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T21:26:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:56,725 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T21:26:56,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T21:26:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111719b87e7321044981819b77b7ac8afdc6_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878816724/Put/seqid=0 2024-11-17T21:26:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:56,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T21:26:56,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:56,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:56,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:56,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:56,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:56,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742445_1621 (size=22618) 2024-11-17T21:26:56,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878876783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:56,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878876785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:56,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878876790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:56,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:56,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:56,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:56,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878876901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:56,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878876901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:56,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878876901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:56,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T21:26:56,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:56,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:56,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:56,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:56,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T21:26:56,990 INFO [Thread-2395 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2395. 2024-11-17T21:26:56,990 DEBUG [Thread-2395 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:56,990 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:56,990 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:56,990 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:56,994 INFO [Thread-2393 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2393. 2024-11-17T21:26:56,994 DEBUG [Thread-2393 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:56,994 DEBUG [Thread-2393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:56,994 DEBUG [Thread-2393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:56,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:56,994 INFO [Thread-2389 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2389. 2024-11-17T21:26:56,994 DEBUG [Thread-2389 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:56,994 DEBUG [Thread-2389 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:56,994 DEBUG [Thread-2389 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:56,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:56,996 INFO [Thread-2387 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2387. 2024-11-17T21:26:56,996 DEBUG [Thread-2387 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:56,996 DEBUG [Thread-2387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:56,996 DEBUG [Thread-2387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:56,996 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:56,997 INFO [Thread-2391 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2391. 2024-11-17T21:26:56,997 DEBUG [Thread-2391 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:56,997 DEBUG [Thread-2391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:56,997 DEBUG [Thread-2391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:56,997 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:57,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T21:26:57,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:57,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:57,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:57,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:57,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878877111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878877111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:57,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878877111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:57,190 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:57,193 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111719b87e7321044981819b77b7ac8afdc6_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111719b87e7321044981819b77b7ac8afdc6_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:57,193 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/a5dd3cc361d5490eaa9622bcf3331229, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:57,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/a5dd3cc361d5490eaa9622bcf3331229 is 175, key is test_row_0/A:col10/1731878816724/Put/seqid=0 2024-11-17T21:26:57,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742446_1622 (size=66023) 2024-11-17T21:26:57,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T21:26:57,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:57,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:57,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:57,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T21:26:57,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T21:26:57,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:57,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:57,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:57,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:57,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:57,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878877421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878877421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878877421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:57,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T21:26:57,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:57,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:57,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:57,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,597 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/a5dd3cc361d5490eaa9622bcf3331229 2024-11-17T21:26:57,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/5a093c32dd654eb2b5b9ebcb15ec7e8d is 50, key is test_row_0/B:col10/1731878816724/Put/seqid=0 2024-11-17T21:26:57,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742447_1623 (size=12301) 2024-11-17T21:26:57,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T21:26:57,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:57,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:57,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:57,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T21:26:57,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T21:26:57,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:57,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:57,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:57,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:57,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:57,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52178 deadline: 1731878877931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:57,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52104 deadline: 1731878877931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:57,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T21:26:57,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52106 deadline: 1731878877931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 2024-11-17T21:26:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T21:26:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., hostname=a313eea8709e,46445,1731878642828, seqNum=5 because the exception is null or not the one we care about 2024-11-17T21:26:57,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T21:26:57,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:57,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:57,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:57,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:57,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:58,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/5a093c32dd654eb2b5b9ebcb15ec7e8d 2024-11-17T21:26:58,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/4bf809bc68ea480eb2c253abf9eb1aaa is 50, key is test_row_0/C:col10/1731878816724/Put/seqid=0 2024-11-17T21:26:58,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742448_1624 (size=12301) 2024-11-17T21:26:58,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T21:26:58,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:58,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:58,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:58,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:58,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:58,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:58,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T21:26:58,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:58,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:58,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:58,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:58,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T21:26:58,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/4bf809bc68ea480eb2c253abf9eb1aaa 2024-11-17T21:26:58,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/a5dd3cc361d5490eaa9622bcf3331229 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/a5dd3cc361d5490eaa9622bcf3331229 2024-11-17T21:26:58,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/a5dd3cc361d5490eaa9622bcf3331229, entries=350, sequenceid=324, filesize=64.5 K 2024-11-17T21:26:58,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/5a093c32dd654eb2b5b9ebcb15ec7e8d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/5a093c32dd654eb2b5b9ebcb15ec7e8d 2024-11-17T21:26:58,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/5a093c32dd654eb2b5b9ebcb15ec7e8d, entries=150, sequenceid=324, filesize=12.0 K 2024-11-17T21:26:58,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/4bf809bc68ea480eb2c253abf9eb1aaa as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/4bf809bc68ea480eb2c253abf9eb1aaa 2024-11-17T21:26:58,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/4bf809bc68ea480eb2c253abf9eb1aaa, entries=150, sequenceid=324, filesize=12.0 K 2024-11-17T21:26:58,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a21acdaa0aabfb075e55b063d1c8e884 in 1703ms, sequenceid=324, compaction requested=true 2024-11-17T21:26:58,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:58,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T21:26:58,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:58,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T21:26:58,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:58,429 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:58,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a21acdaa0aabfb075e55b063d1c8e884:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T21:26:58,429 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:58,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:58,429 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129215 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:58,429 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:58,430 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/A is initiating minor compaction (all files) 2024-11-17T21:26:58,430 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/B is initiating minor compaction (all files) 2024-11-17T21:26:58,430 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/A in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:58,430 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/B in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:58,430 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/f5c497a811384d29816b6c133a6b25d7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/6f3d3387ad3e478fb1b55f9314074b96, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/5a093c32dd654eb2b5b9ebcb15ec7e8d] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=36.7 K 2024-11-17T21:26:58,430 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/379ab3072efa4a9680c87085318282c7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/9e42292fa7fd43adafce78b03be762a9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/a5dd3cc361d5490eaa9622bcf3331229] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=126.2 K 2024-11-17T21:26:58,430 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:58,430 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. files: [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/379ab3072efa4a9680c87085318282c7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/9e42292fa7fd43adafce78b03be762a9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/a5dd3cc361d5490eaa9622bcf3331229] 2024-11-17T21:26:58,430 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting f5c497a811384d29816b6c133a6b25d7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731878813843 2024-11-17T21:26:58,430 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 379ab3072efa4a9680c87085318282c7, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731878813843 2024-11-17T21:26:58,430 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f3d3387ad3e478fb1b55f9314074b96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731878814514 2024-11-17T21:26:58,430 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e42292fa7fd43adafce78b03be762a9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731878814514 2024-11-17T21:26:58,430 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a093c32dd654eb2b5b9ebcb15ec7e8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731878816713 2024-11-17T21:26:58,430 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5dd3cc361d5490eaa9622bcf3331229, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731878816689 2024-11-17T21:26:58,435 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#B#compaction#521 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:58,436 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/dde7f64292c24883a05880480a2732ab is 50, key is test_row_0/B:col10/1731878816724/Put/seqid=0 2024-11-17T21:26:58,436 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:58,438 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411173b4ee0d03c2a497c81d4ff231391b6e4_a21acdaa0aabfb075e55b063d1c8e884 store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:58,440 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411173b4ee0d03c2a497c81d4ff231391b6e4_a21acdaa0aabfb075e55b063d1c8e884, store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:58,440 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173b4ee0d03c2a497c81d4ff231391b6e4_a21acdaa0aabfb075e55b063d1c8e884 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:58,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742449_1625 (size=13085) 2024-11-17T21:26:58,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742450_1626 (size=4469) 2024-11-17T21:26:58,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46445 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T21:26:58,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:58,448 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T21:26:58,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bb2a114dc3914f8c9829c88eb6f1bde0_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878816778/Put/seqid=0 2024-11-17T21:26:58,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742451_1627 (size=12454) 2024-11-17T21:26:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T21:26:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46445 {}] regionserver.HRegion(8855): Flush requested on a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:58,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. as already flushing 2024-11-17T21:26:58,832 INFO [Thread-2382 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2382. 2024-11-17T21:26:58,832 DEBUG [Thread-2382 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:58,833 DEBUG [Thread-2382 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:58,833 DEBUG [Thread-2382 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:58,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:58,844 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#A#compaction#522 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:58,845 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/4124673bfeb04357a991a12763807522 is 175, key is test_row_0/A:col10/1731878816724/Put/seqid=0 2024-11-17T21:26:58,845 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/dde7f64292c24883a05880480a2732ab as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/dde7f64292c24883a05880480a2732ab 2024-11-17T21:26:58,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742452_1628 (size=32039) 2024-11-17T21:26:58,849 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/B of a21acdaa0aabfb075e55b063d1c8e884 into dde7f64292c24883a05880480a2732ab(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:58,849 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:58,849 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/B, priority=13, startTime=1731878818429; duration=0sec 2024-11-17T21:26:58,849 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T21:26:58,849 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:B 2024-11-17T21:26:58,849 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T21:26:58,850 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T21:26:58,850 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1541): a21acdaa0aabfb075e55b063d1c8e884/C is initiating minor compaction (all files) 2024-11-17T21:26:58,850 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a21acdaa0aabfb075e55b063d1c8e884/C in TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:26:58,850 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/0ac9a230b8b64c86a1ea32588c59d756, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/a00941d9d87f44d1888191d92505f171, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/4bf809bc68ea480eb2c253abf9eb1aaa] into tmpdir=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp, totalSize=36.7 K 2024-11-17T21:26:58,850 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ac9a230b8b64c86a1ea32588c59d756, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731878813843 2024-11-17T21:26:58,850 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting a00941d9d87f44d1888191d92505f171, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731878814514 2024-11-17T21:26:58,850 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bf809bc68ea480eb2c253abf9eb1aaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731878816713 2024-11-17T21:26:58,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:26:58,856 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a21acdaa0aabfb075e55b063d1c8e884#C#compaction#524 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T21:26:58,857 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/b9baeb11f3034fcfa62854d0ac02adea is 50, key is test_row_0/C:col10/1731878816724/Put/seqid=0 2024-11-17T21:26:58,859 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bb2a114dc3914f8c9829c88eb6f1bde0_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bb2a114dc3914f8c9829c88eb6f1bde0_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:26:58,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/c1f3c515a82f4cc4b111eb1dfeb86730, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:26:58,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/c1f3c515a82f4cc4b111eb1dfeb86730 is 175, key is test_row_0/A:col10/1731878816778/Put/seqid=0 2024-11-17T21:26:58,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742453_1629 (size=13085) 2024-11-17T21:26:58,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742454_1630 (size=31255) 2024-11-17T21:26:58,942 INFO [Thread-2376 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2376. 2024-11-17T21:26:58,943 DEBUG [Thread-2376 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:58,943 DEBUG [Thread-2376 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:58,943 DEBUG [Thread-2376 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:58,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:58,943 INFO [Thread-2378 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2378. 2024-11-17T21:26:58,943 DEBUG [Thread-2378 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:58,943 DEBUG [Thread-2378 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:58,943 DEBUG [Thread-2378 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:58,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:58,952 INFO [Thread-2380 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2380. 2024-11-17T21:26:58,952 DEBUG [Thread-2380 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:26:58,952 DEBUG [Thread-2380 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:58,952 DEBUG [Thread-2380 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:26:58,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:26:59,252 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/4124673bfeb04357a991a12763807522 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/4124673bfeb04357a991a12763807522 2024-11-17T21:26:59,255 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/A of a21acdaa0aabfb075e55b063d1c8e884 into 4124673bfeb04357a991a12763807522(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:59,255 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:59,255 INFO [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/A, priority=13, startTime=1731878818429; duration=0sec 2024-11-17T21:26:59,255 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:59,255 DEBUG [RS:0;a313eea8709e:46445-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:A 2024-11-17T21:26:59,264 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/c1f3c515a82f4cc4b111eb1dfeb86730 2024-11-17T21:26:59,268 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/b9baeb11f3034fcfa62854d0ac02adea as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/b9baeb11f3034fcfa62854d0ac02adea 2024-11-17T21:26:59,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/aae664c4314c44b1bb2d24d48e068af4 is 50, key is test_row_0/B:col10/1731878816778/Put/seqid=0 2024-11-17T21:26:59,272 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a21acdaa0aabfb075e55b063d1c8e884/C of a21acdaa0aabfb075e55b063d1c8e884 into b9baeb11f3034fcfa62854d0ac02adea(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T21:26:59,272 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:26:59,272 INFO [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., storeName=a21acdaa0aabfb075e55b063d1c8e884/C, priority=13, startTime=1731878818429; duration=0sec 2024-11-17T21:26:59,272 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T21:26:59,272 DEBUG [RS:0;a313eea8709e:46445-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a21acdaa0aabfb075e55b063d1c8e884:C 2024-11-17T21:26:59,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742455_1631 (size=12301) 2024-11-17T21:26:59,673 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/aae664c4314c44b1bb2d24d48e068af4 2024-11-17T21:26:59,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/62989398707143278b4a1bac1da2da85 is 50, key is test_row_0/C:col10/1731878816778/Put/seqid=0 2024-11-17T21:26:59,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742456_1632 (size=12301) 2024-11-17T21:27:00,086 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/62989398707143278b4a1bac1da2da85 2024-11-17T21:27:00,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/c1f3c515a82f4cc4b111eb1dfeb86730 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/c1f3c515a82f4cc4b111eb1dfeb86730 2024-11-17T21:27:00,092 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/c1f3c515a82f4cc4b111eb1dfeb86730, entries=150, sequenceid=349, filesize=30.5 K 2024-11-17T21:27:00,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/aae664c4314c44b1bb2d24d48e068af4 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/aae664c4314c44b1bb2d24d48e068af4 2024-11-17T21:27:00,095 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/aae664c4314c44b1bb2d24d48e068af4, entries=150, sequenceid=349, filesize=12.0 K 2024-11-17T21:27:00,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/62989398707143278b4a1bac1da2da85 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/62989398707143278b4a1bac1da2da85 2024-11-17T21:27:00,098 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/62989398707143278b4a1bac1da2da85, entries=150, sequenceid=349, filesize=12.0 K 2024-11-17T21:27:00,098 INFO [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=26.84 KB/27480 for a21acdaa0aabfb075e55b063d1c8e884 in 1650ms, sequenceid=349, compaction requested=false 2024-11-17T21:27:00,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for a21acdaa0aabfb075e55b063d1c8e884: 2024-11-17T21:27:00,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:27:00,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a313eea8709e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-17T21:27:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-17T21:27:00,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-17T21:27:00,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4850 sec 2024-11-17T21:27:00,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 3.4890 sec 2024-11-17T21:27:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T21:27:00,781 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:27:01,090 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T21:27:05,442 INFO [Thread-2384 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2384. 2024-11-17T21:27:05,443 DEBUG [Thread-2384 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T21:27:05,443 DEBUG [Thread-2384 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:27:05,443 DEBUG [Thread-2384 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:27:05,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:27:05,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T21:27:05,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-17T21:27:05,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-17T21:27:05,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-17T21:27:05,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-17T21:27:05,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-17T21:27:05,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T21:27:05,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3835 2024-11-17T21:27:05,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3980 2024-11-17T21:27:05,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3726 2024-11-17T21:27:05,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3858 2024-11-17T21:27:05,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4002 2024-11-17T21:27:05,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T21:27:05,443 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T21:27:05,443 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T21:27:05,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:27:05,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:27:05,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:27:05,444 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T21:27:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T21:27:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T21:27:05,447 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878825447"}]},"ts":"1731878825447"} 2024-11-17T21:27:05,448 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T21:27:05,448 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T21:27:05,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T21:27:05,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a21acdaa0aabfb075e55b063d1c8e884, UNASSIGN}] 2024-11-17T21:27:05,451 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a21acdaa0aabfb075e55b063d1c8e884, UNASSIGN 2024-11-17T21:27:05,451 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=a21acdaa0aabfb075e55b063d1c8e884, regionState=CLOSING, regionLocation=a313eea8709e,46445,1731878642828 2024-11-17T21:27:05,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a21acdaa0aabfb075e55b063d1c8e884, UNASSIGN because future has completed 2024-11-17T21:27:05,452 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T21:27:05,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828}] 2024-11-17T21:27:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T21:27:05,605 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:05,605 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T21:27:05,605 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing a21acdaa0aabfb075e55b063d1c8e884, disabling compactions & flushes 2024-11-17T21:27:05,605 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:27:05,605 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:27:05,605 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. after waiting 0 ms 2024-11-17T21:27:05,605 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:27:05,605 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing a21acdaa0aabfb075e55b063d1c8e884 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-17T21:27:05,610 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b496d7b2c8e34f4b8a503dcfcf29efae_a21acdaa0aabfb075e55b063d1c8e884 is 50, key is test_row_0/A:col10/1731878818951/Put/seqid=0 2024-11-17T21:27:05,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742457_1633 (size=9914) 2024-11-17T21:27:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T21:27:06,013 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T21:27:06,016 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b496d7b2c8e34f4b8a503dcfcf29efae_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b496d7b2c8e34f4b8a503dcfcf29efae_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:06,017 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/b04affc6197e426a960d3a196ec654b8, store: [table=TestAcidGuarantees family=A region=a21acdaa0aabfb075e55b063d1c8e884] 2024-11-17T21:27:06,017 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/b04affc6197e426a960d3a196ec654b8 is 175, key is test_row_0/A:col10/1731878818951/Put/seqid=0 2024-11-17T21:27:06,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742458_1634 (size=22561) 2024-11-17T21:27:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T21:27:06,421 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/b04affc6197e426a960d3a196ec654b8 2024-11-17T21:27:06,426 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/9df42e91f9fa4ce997d7d0512455027f is 50, key is test_row_0/B:col10/1731878818951/Put/seqid=0 2024-11-17T21:27:06,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742459_1635 (size=9857) 2024-11-17T21:27:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T21:27:06,829 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/9df42e91f9fa4ce997d7d0512455027f 2024-11-17T21:27:06,835 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/573e9aaca0f946a796f60eb320ab13da is 50, key is test_row_0/C:col10/1731878818951/Put/seqid=0 2024-11-17T21:27:06,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742460_1636 (size=9857) 2024-11-17T21:27:07,238 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/573e9aaca0f946a796f60eb320ab13da 2024-11-17T21:27:07,242 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/A/b04affc6197e426a960d3a196ec654b8 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/b04affc6197e426a960d3a196ec654b8 2024-11-17T21:27:07,244 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/b04affc6197e426a960d3a196ec654b8, entries=100, sequenceid=360, filesize=22.0 K 2024-11-17T21:27:07,245 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/B/9df42e91f9fa4ce997d7d0512455027f as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/9df42e91f9fa4ce997d7d0512455027f 2024-11-17T21:27:07,248 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/9df42e91f9fa4ce997d7d0512455027f, entries=100, sequenceid=360, filesize=9.6 K 2024-11-17T21:27:07,248 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/.tmp/C/573e9aaca0f946a796f60eb320ab13da as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/573e9aaca0f946a796f60eb320ab13da 2024-11-17T21:27:07,251 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/573e9aaca0f946a796f60eb320ab13da, entries=100, sequenceid=360, filesize=9.6 K 2024-11-17T21:27:07,252 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a21acdaa0aabfb075e55b063d1c8e884 in 1646ms, sequenceid=360, compaction requested=true 2024-11-17T21:27:07,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/cd1b5c760b504d7486d5e30822011fe5, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dc0ed2d2f5dd4789b6523b70e8f9f7f8, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dcc235bb8e6c4cffaf1cb8885b7bb6f0, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/db67ad073c9f4f54b7b4a44675bf303f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/3586f00f4aa144da9f3bd75f8e4bd018, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/f969075e0f5f4d3c9caf44eeedbf7f14, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/c484faa8dee04bf08eefb06f6f16c03d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/cfe8aaa223c54cb2b55e50cd3a8db50b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/80f8b13bb2ab4bd79514cfbba05ff017, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/c5f24d7cc2b04ab8ad1d728b86a957b4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/2a62c88260764c468d86cca83f86a54d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dc222d5043814b9f8626a32da3f130cd, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/7f43b026e57a4dd197a46dc4181ea2ba, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/1a487c44355e49a19c6b3242c3ca080d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/2e02c5d7786a4b05b0beace293888850, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/a6f98bc3cdfe483eaa8144d5a52787bc, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/7d5dbfd9a5e945b3853f128f4e58dc3a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/52a72647b37941a8af45fd3541f1129e, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/9b662bc1ca1d48fc8bdc9a11b80d1494, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/15aeb7c2c5c94c1d9d2f095b8cb412ae, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/379ab3072efa4a9680c87085318282c7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/9e42292fa7fd43adafce78b03be762a9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/a5dd3cc361d5490eaa9622bcf3331229] to archive 2024-11-17T21:27:07,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:27:07,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/cd1b5c760b504d7486d5e30822011fe5 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/cd1b5c760b504d7486d5e30822011fe5 2024-11-17T21:27:07,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dc0ed2d2f5dd4789b6523b70e8f9f7f8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dc0ed2d2f5dd4789b6523b70e8f9f7f8 2024-11-17T21:27:07,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dcc235bb8e6c4cffaf1cb8885b7bb6f0 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dcc235bb8e6c4cffaf1cb8885b7bb6f0 2024-11-17T21:27:07,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/db67ad073c9f4f54b7b4a44675bf303f to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/db67ad073c9f4f54b7b4a44675bf303f 2024-11-17T21:27:07,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/3586f00f4aa144da9f3bd75f8e4bd018 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/3586f00f4aa144da9f3bd75f8e4bd018 2024-11-17T21:27:07,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/f969075e0f5f4d3c9caf44eeedbf7f14 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/f969075e0f5f4d3c9caf44eeedbf7f14 2024-11-17T21:27:07,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/c484faa8dee04bf08eefb06f6f16c03d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/c484faa8dee04bf08eefb06f6f16c03d 2024-11-17T21:27:07,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/cfe8aaa223c54cb2b55e50cd3a8db50b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/cfe8aaa223c54cb2b55e50cd3a8db50b 2024-11-17T21:27:07,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/80f8b13bb2ab4bd79514cfbba05ff017 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/80f8b13bb2ab4bd79514cfbba05ff017 2024-11-17T21:27:07,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/c5f24d7cc2b04ab8ad1d728b86a957b4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/c5f24d7cc2b04ab8ad1d728b86a957b4 2024-11-17T21:27:07,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/2a62c88260764c468d86cca83f86a54d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/2a62c88260764c468d86cca83f86a54d 2024-11-17T21:27:07,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dc222d5043814b9f8626a32da3f130cd to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/dc222d5043814b9f8626a32da3f130cd 2024-11-17T21:27:07,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/7f43b026e57a4dd197a46dc4181ea2ba to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/7f43b026e57a4dd197a46dc4181ea2ba 2024-11-17T21:27:07,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/1a487c44355e49a19c6b3242c3ca080d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/1a487c44355e49a19c6b3242c3ca080d 2024-11-17T21:27:07,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/2e02c5d7786a4b05b0beace293888850 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/2e02c5d7786a4b05b0beace293888850 2024-11-17T21:27:07,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/a6f98bc3cdfe483eaa8144d5a52787bc to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/a6f98bc3cdfe483eaa8144d5a52787bc 2024-11-17T21:27:07,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/7d5dbfd9a5e945b3853f128f4e58dc3a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/7d5dbfd9a5e945b3853f128f4e58dc3a 2024-11-17T21:27:07,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/52a72647b37941a8af45fd3541f1129e to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/52a72647b37941a8af45fd3541f1129e 2024-11-17T21:27:07,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/9b662bc1ca1d48fc8bdc9a11b80d1494 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/9b662bc1ca1d48fc8bdc9a11b80d1494 2024-11-17T21:27:07,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/15aeb7c2c5c94c1d9d2f095b8cb412ae to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/15aeb7c2c5c94c1d9d2f095b8cb412ae 2024-11-17T21:27:07,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/379ab3072efa4a9680c87085318282c7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/379ab3072efa4a9680c87085318282c7 2024-11-17T21:27:07,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/9e42292fa7fd43adafce78b03be762a9 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/9e42292fa7fd43adafce78b03be762a9 2024-11-17T21:27:07,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/a5dd3cc361d5490eaa9622bcf3331229 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/a5dd3cc361d5490eaa9622bcf3331229 2024-11-17T21:27:07,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/bab509df0d834661a52d047498867e9b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/0c2f07ef4ec94b5db2dcfcf340b846ed, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/1f711e11c3854e03a19ec87476db3188, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/51e592e43b9c4d41a8fa628806c55ea9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/84af635efa0a4c3bb597920716a8814d, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/4235733e698046fc83174f8e9db684d0, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/db07635a8020470b844701e3f9f01685, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/3b40d34c79f34ab9b07c7c1b80a1c593, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/c22845a4c59f48d9a2f298ac3e0e50bf, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/aa700c464833434f87e400b407015a5b, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/a896f00f5eac40c6a350229d0d1e15b2, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/208ac44c26784fb38c170677bc98ce84, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/646a84247df0445d910e64cfca05b0b4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/9a6ef4a4dd3d4f21ab3111a6ffced507, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/1e77b6b570b84cd1bb8a8f2fab296ad9, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/f053dea396f24438a31faf53aa86ba53, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/e0a087501bd3465bbefb68e3e9ae5896, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/d122d18498e745d1abf0ef239c9c8017, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/5ba7bdf9bb4a484c83c839d6eb44a518, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/f5c497a811384d29816b6c133a6b25d7, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/8d3a8a007cb141d5b191bfb23907d207, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/6f3d3387ad3e478fb1b55f9314074b96, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/5a093c32dd654eb2b5b9ebcb15ec7e8d] to archive 2024-11-17T21:27:07,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:27:07,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/bab509df0d834661a52d047498867e9b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/bab509df0d834661a52d047498867e9b 2024-11-17T21:27:07,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/0c2f07ef4ec94b5db2dcfcf340b846ed to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/0c2f07ef4ec94b5db2dcfcf340b846ed 2024-11-17T21:27:07,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/1f711e11c3854e03a19ec87476db3188 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/1f711e11c3854e03a19ec87476db3188 2024-11-17T21:27:07,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/51e592e43b9c4d41a8fa628806c55ea9 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/51e592e43b9c4d41a8fa628806c55ea9 2024-11-17T21:27:07,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/84af635efa0a4c3bb597920716a8814d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/84af635efa0a4c3bb597920716a8814d 2024-11-17T21:27:07,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/4235733e698046fc83174f8e9db684d0 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/4235733e698046fc83174f8e9db684d0 2024-11-17T21:27:07,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/db07635a8020470b844701e3f9f01685 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/db07635a8020470b844701e3f9f01685 2024-11-17T21:27:07,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/3b40d34c79f34ab9b07c7c1b80a1c593 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/3b40d34c79f34ab9b07c7c1b80a1c593 2024-11-17T21:27:07,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/c22845a4c59f48d9a2f298ac3e0e50bf to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/c22845a4c59f48d9a2f298ac3e0e50bf 2024-11-17T21:27:07,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/aa700c464833434f87e400b407015a5b to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/aa700c464833434f87e400b407015a5b 2024-11-17T21:27:07,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/a896f00f5eac40c6a350229d0d1e15b2 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/a896f00f5eac40c6a350229d0d1e15b2 2024-11-17T21:27:07,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/208ac44c26784fb38c170677bc98ce84 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/208ac44c26784fb38c170677bc98ce84 2024-11-17T21:27:07,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/646a84247df0445d910e64cfca05b0b4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/646a84247df0445d910e64cfca05b0b4 2024-11-17T21:27:07,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/9a6ef4a4dd3d4f21ab3111a6ffced507 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/9a6ef4a4dd3d4f21ab3111a6ffced507 2024-11-17T21:27:07,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/1e77b6b570b84cd1bb8a8f2fab296ad9 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/1e77b6b570b84cd1bb8a8f2fab296ad9 2024-11-17T21:27:07,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/f053dea396f24438a31faf53aa86ba53 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/f053dea396f24438a31faf53aa86ba53 2024-11-17T21:27:07,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/e0a087501bd3465bbefb68e3e9ae5896 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/e0a087501bd3465bbefb68e3e9ae5896 2024-11-17T21:27:07,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/d122d18498e745d1abf0ef239c9c8017 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/d122d18498e745d1abf0ef239c9c8017 2024-11-17T21:27:07,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/5ba7bdf9bb4a484c83c839d6eb44a518 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/5ba7bdf9bb4a484c83c839d6eb44a518 2024-11-17T21:27:07,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/f5c497a811384d29816b6c133a6b25d7 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/f5c497a811384d29816b6c133a6b25d7 2024-11-17T21:27:07,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/8d3a8a007cb141d5b191bfb23907d207 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/8d3a8a007cb141d5b191bfb23907d207 2024-11-17T21:27:07,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/6f3d3387ad3e478fb1b55f9314074b96 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/6f3d3387ad3e478fb1b55f9314074b96 2024-11-17T21:27:07,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/5a093c32dd654eb2b5b9ebcb15ec7e8d to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/5a093c32dd654eb2b5b9ebcb15ec7e8d 2024-11-17T21:27:07,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/23d6b0bd81cc4ba2b2e7fb45342255c6, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/530c5ca2103d4e6298fd34bec4e38947, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/0a384390bcbe4dd9bd9c8166641be8fe, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/e219dcde6a3f442ca757d618a180c6ec, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/8732b6c6b96e4aa7964266e61d1771ee, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/cf5ed912578f4c22b23e016eb0f28eff, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/ea3ce61f2f7a40e394c1e938e7f1be47, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/055173f5a2b5461e89294e18cf96e424, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/49b68ca1cc6042b1bc50ca96478b5823, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/330df5d5ec984cf3830795cfb01a14e2, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/4b3d3a4ab63f486cae4ce2d1c589270f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/9ae4cde614f54cd6a8711df93894d556, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/b7fb4e735d0248488c23bcbe75103d9a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/18dbd3ae869040b2ba60898f63798c96, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/a7187495fb434b75a130898c041fe291, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/26ab53aa9df5479fa05d37d8ee6fd49a, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/3b4e6454c5f44936836818ec2f562b30, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/1a61996b615e4e8bb27b5a3db099bc7f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/d6e794dc80724b72ad536a2e5e69c2e4, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/0ac9a230b8b64c86a1ea32588c59d756, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/2da40d962f2e48a0b36c26828924c99f, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/a00941d9d87f44d1888191d92505f171, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/4bf809bc68ea480eb2c253abf9eb1aaa] to archive 2024-11-17T21:27:07,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T21:27:07,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/23d6b0bd81cc4ba2b2e7fb45342255c6 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/23d6b0bd81cc4ba2b2e7fb45342255c6 2024-11-17T21:27:07,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/530c5ca2103d4e6298fd34bec4e38947 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/530c5ca2103d4e6298fd34bec4e38947 2024-11-17T21:27:07,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/0a384390bcbe4dd9bd9c8166641be8fe to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/0a384390bcbe4dd9bd9c8166641be8fe 2024-11-17T21:27:07,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/e219dcde6a3f442ca757d618a180c6ec to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/e219dcde6a3f442ca757d618a180c6ec 2024-11-17T21:27:07,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/8732b6c6b96e4aa7964266e61d1771ee to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/8732b6c6b96e4aa7964266e61d1771ee 2024-11-17T21:27:07,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/cf5ed912578f4c22b23e016eb0f28eff to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/cf5ed912578f4c22b23e016eb0f28eff 2024-11-17T21:27:07,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/ea3ce61f2f7a40e394c1e938e7f1be47 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/ea3ce61f2f7a40e394c1e938e7f1be47 2024-11-17T21:27:07,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/055173f5a2b5461e89294e18cf96e424 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/055173f5a2b5461e89294e18cf96e424 2024-11-17T21:27:07,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/49b68ca1cc6042b1bc50ca96478b5823 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/49b68ca1cc6042b1bc50ca96478b5823 2024-11-17T21:27:07,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/330df5d5ec984cf3830795cfb01a14e2 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/330df5d5ec984cf3830795cfb01a14e2 2024-11-17T21:27:07,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/4b3d3a4ab63f486cae4ce2d1c589270f to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/4b3d3a4ab63f486cae4ce2d1c589270f 2024-11-17T21:27:07,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/9ae4cde614f54cd6a8711df93894d556 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/9ae4cde614f54cd6a8711df93894d556 2024-11-17T21:27:07,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/b7fb4e735d0248488c23bcbe75103d9a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/b7fb4e735d0248488c23bcbe75103d9a 2024-11-17T21:27:07,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/18dbd3ae869040b2ba60898f63798c96 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/18dbd3ae869040b2ba60898f63798c96 2024-11-17T21:27:07,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/a7187495fb434b75a130898c041fe291 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/a7187495fb434b75a130898c041fe291 2024-11-17T21:27:07,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/26ab53aa9df5479fa05d37d8ee6fd49a to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/26ab53aa9df5479fa05d37d8ee6fd49a 2024-11-17T21:27:07,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/3b4e6454c5f44936836818ec2f562b30 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/3b4e6454c5f44936836818ec2f562b30 2024-11-17T21:27:07,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/1a61996b615e4e8bb27b5a3db099bc7f to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/1a61996b615e4e8bb27b5a3db099bc7f 2024-11-17T21:27:07,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/d6e794dc80724b72ad536a2e5e69c2e4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/d6e794dc80724b72ad536a2e5e69c2e4 2024-11-17T21:27:07,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/0ac9a230b8b64c86a1ea32588c59d756 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/0ac9a230b8b64c86a1ea32588c59d756 2024-11-17T21:27:07,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/2da40d962f2e48a0b36c26828924c99f to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/2da40d962f2e48a0b36c26828924c99f 2024-11-17T21:27:07,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/a00941d9d87f44d1888191d92505f171 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/a00941d9d87f44d1888191d92505f171 2024-11-17T21:27:07,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/4bf809bc68ea480eb2c253abf9eb1aaa to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/4bf809bc68ea480eb2c253abf9eb1aaa 2024-11-17T21:27:07,310 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/recovered.edits/363.seqid, newMaxSeqId=363, maxSeqId=4 2024-11-17T21:27:07,310 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. 2024-11-17T21:27:07,310 DEBUG [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for a21acdaa0aabfb075e55b063d1c8e884: Waiting for close lock at 1731878825605Running coprocessor pre-close hooks at 1731878825605Disabling compacts and flushes for region at 1731878825605Disabling writes for close at 1731878825605Obtaining lock to block concurrent updates at 1731878825605Preparing flush snapshotting stores in a21acdaa0aabfb075e55b063d1c8e884 at 1731878825605Finished memstore snapshotting TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731878825605Flushing stores of TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884. at 1731878825606 (+1 ms)Flushing a21acdaa0aabfb075e55b063d1c8e884/A: creating writer at 1731878825606Flushing mob file a21acdaa0aabfb075e55b063d1c8e884/A: appending metadata at 1731878825610 (+4 ms)Flushing mob file a21acdaa0aabfb075e55b063d1c8e884/A: closing flushed file at 1731878825610Flushing a21acdaa0aabfb075e55b063d1c8e884/A: appending metadata at 1731878826017 (+407 ms)Flushing a21acdaa0aabfb075e55b063d1c8e884/A: closing flushed file at 1731878826017Flushing a21acdaa0aabfb075e55b063d1c8e884/B: creating writer at 1731878826424 (+407 ms)Flushing a21acdaa0aabfb075e55b063d1c8e884/B: appending metadata at 1731878826426 (+2 ms)Flushing a21acdaa0aabfb075e55b063d1c8e884/B: closing flushed file at 1731878826426Flushing a21acdaa0aabfb075e55b063d1c8e884/C: creating writer at 1731878826832 (+406 ms)Flushing a21acdaa0aabfb075e55b063d1c8e884/C: appending metadata at 1731878826834 (+2 ms)Flushing a21acdaa0aabfb075e55b063d1c8e884/C: closing flushed file at 1731878826834Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56233f58: reopening flushed file at 1731878827241 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bff79a: reopening flushed file at 1731878827244 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d9e0b80: reopening flushed file at 1731878827248 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a21acdaa0aabfb075e55b063d1c8e884 in 1646ms, sequenceid=360, compaction requested=true at 1731878827252 (+4 ms)Writing region close event to WAL at 1731878827308 (+56 ms)Running coprocessor post-close hooks at 1731878827310 (+2 ms)Closed at 1731878827310 2024-11-17T21:27:07,312 INFO [RS_CLOSE_REGION-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,313 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=a21acdaa0aabfb075e55b063d1c8e884, regionState=CLOSED 2024-11-17T21:27:07,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 because future has completed 2024-11-17T21:27:07,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-17T21:27:07,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure a21acdaa0aabfb075e55b063d1c8e884, server=a313eea8709e,46445,1731878642828 in 1.8620 sec 2024-11-17T21:27:07,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-17T21:27:07,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a21acdaa0aabfb075e55b063d1c8e884, UNASSIGN in 1.8660 sec 2024-11-17T21:27:07,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-17T21:27:07,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8690 sec 2024-11-17T21:27:07,320 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731878827319"}]},"ts":"1731878827319"} 2024-11-17T21:27:07,320 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T21:27:07,320 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T21:27:07,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8770 sec 2024-11-17T21:27:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T21:27:07,591 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:27:07,592 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T21:27:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:27:07,593 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:27:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T21:27:07,594 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:27:07,595 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,597 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C, FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/recovered.edits] 2024-11-17T21:27:07,598 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/4124673bfeb04357a991a12763807522 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/4124673bfeb04357a991a12763807522 2024-11-17T21:27:07,599 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/b04affc6197e426a960d3a196ec654b8 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/b04affc6197e426a960d3a196ec654b8 2024-11-17T21:27:07,600 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/c1f3c515a82f4cc4b111eb1dfeb86730 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/A/c1f3c515a82f4cc4b111eb1dfeb86730 2024-11-17T21:27:07,601 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/9df42e91f9fa4ce997d7d0512455027f to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/9df42e91f9fa4ce997d7d0512455027f 2024-11-17T21:27:07,602 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/aae664c4314c44b1bb2d24d48e068af4 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/aae664c4314c44b1bb2d24d48e068af4 2024-11-17T21:27:07,602 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/dde7f64292c24883a05880480a2732ab to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/B/dde7f64292c24883a05880480a2732ab 2024-11-17T21:27:07,604 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/573e9aaca0f946a796f60eb320ab13da to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/573e9aaca0f946a796f60eb320ab13da 2024-11-17T21:27:07,604 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/62989398707143278b4a1bac1da2da85 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/62989398707143278b4a1bac1da2da85 2024-11-17T21:27:07,605 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/b9baeb11f3034fcfa62854d0ac02adea to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/C/b9baeb11f3034fcfa62854d0ac02adea 2024-11-17T21:27:07,606 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/recovered.edits/363.seqid to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884/recovered.edits/363.seqid 2024-11-17T21:27:07,607 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/default/TestAcidGuarantees/a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,607 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T21:27:07,607 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T21:27:07,608 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T21:27:07,609 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171356ca3c8ea9437891d462517db2dd56_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171356ca3c8ea9437891d462517db2dd56_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,610 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111719339669c89d468783f3b76d5a1c36b0_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111719339669c89d468783f3b76d5a1c36b0_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,611 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111719b87e7321044981819b77b7ac8afdc6_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111719b87e7321044981819b77b7ac8afdc6_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,612 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111739de58b4896b488e9ab8493e531860a4_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111739de58b4896b488e9ab8493e531860a4_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,612 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174684de26870c4add90750ef0c7aa9300_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174684de26870c4add90750ef0c7aa9300_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,613 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117753384e957464af9bbb08cb74135abab_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117753384e957464af9bbb08cb74135abab_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,614 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117804f4ba449af4267b4b572ebd7dffde6_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117804f4ba449af4267b4b572ebd7dffde6_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,615 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111783e98509146b42a2a96a1542c1976cbb_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111783e98509146b42a2a96a1542c1976cbb_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,616 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111786d9b47086d34f1488e19064e462f628_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111786d9b47086d34f1488e19064e462f628_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,616 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111787d78de9ecc44a89b964d6d60060d959_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111787d78de9ecc44a89b964d6d60060d959_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,617 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178f319ce591a04264a73d7a8574749c48_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178f319ce591a04264a73d7a8574749c48_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,618 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179e51dcba7fd64d69a551034f4453f7df_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179e51dcba7fd64d69a551034f4453f7df_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,619 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b496d7b2c8e34f4b8a503dcfcf29efae_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b496d7b2c8e34f4b8a503dcfcf29efae_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,619 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b4d78f7ca4db422a9b70c7915e8e3a30_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b4d78f7ca4db422a9b70c7915e8e3a30_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,620 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bb2a114dc3914f8c9829c88eb6f1bde0_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bb2a114dc3914f8c9829c88eb6f1bde0_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,621 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c6fdf8f1b292403dbcdd0fb577f970b9_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c6fdf8f1b292403dbcdd0fb577f970b9_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,622 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117dfc7b0688dab48d1aee48b241a88ae7c_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117dfc7b0688dab48d1aee48b241a88ae7c_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,623 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f05189b9b3474279ad30b98d78718767_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f05189b9b3474279ad30b98d78718767_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,623 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f32d230e39884c88a3b5fa727b70805f_a21acdaa0aabfb075e55b063d1c8e884 to hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f32d230e39884c88a3b5fa727b70805f_a21acdaa0aabfb075e55b063d1c8e884 2024-11-17T21:27:07,624 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T21:27:07,626 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:27:07,628 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T21:27:07,629 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T21:27:07,630 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:27:07,631 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T21:27:07,631 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731878827631"}]},"ts":"9223372036854775807"} 2024-11-17T21:27:07,632 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T21:27:07,632 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a21acdaa0aabfb075e55b063d1c8e884, NAME => 'TestAcidGuarantees,,1731878793887.a21acdaa0aabfb075e55b063d1c8e884.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T21:27:07,632 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T21:27:07,633 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731878827633"}]},"ts":"9223372036854775807"} 2024-11-17T21:27:07,634 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T21:27:07,634 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T21:27:07,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-17T21:27:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43623 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T21:27:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T21:27:07,702 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T21:27:07,709 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 237), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=427 (was 469), ProcessCount=11 (was 11), AvailableMemoryMB=7949 (was 8001) 2024-11-17T21:27:07,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T21:27:07,709 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T21:27:07,709 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T21:27:07,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:27:07,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:27:07,709 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T21:27:07,709 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-17T21:27:07,709 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1484730631, stopped=false 2024-11-17T21:27:07,710 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a313eea8709e,43623,1731878641945 2024-11-17T21:27:07,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T21:27:07,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T21:27:07,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T21:27:07,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T21:27:07,780 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T21:27:07,781 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T21:27:07,781 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T21:27:07,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:27:07,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T21:27:07,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T21:27:07,781 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a313eea8709e,46445,1731878642828' ***** 2024-11-17T21:27:07,781 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T21:27:07,782 INFO [RS:0;a313eea8709e:46445 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T21:27:07,782 INFO [RS:0;a313eea8709e:46445 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T21:27:07,782 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T21:27:07,782 INFO [RS:0;a313eea8709e:46445 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T21:27:07,782 INFO [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(959): stopping server a313eea8709e,46445,1731878642828 2024-11-17T21:27:07,783 INFO [RS:0;a313eea8709e:46445 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T21:27:07,783 INFO [RS:0;a313eea8709e:46445 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a313eea8709e:46445. 2024-11-17T21:27:07,783 DEBUG [RS:0;a313eea8709e:46445 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T21:27:07,783 DEBUG [RS:0;a313eea8709e:46445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:27:07,783 INFO [RS:0;a313eea8709e:46445 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T21:27:07,783 INFO [RS:0;a313eea8709e:46445 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T21:27:07,783 INFO [RS:0;a313eea8709e:46445 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T21:27:07,783 INFO [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T21:27:07,784 INFO [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T21:27:07,784 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-17T21:27:07,784 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T21:27:07,784 DEBUG [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T21:27:07,784 INFO [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T21:27:07,784 DEBUG [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T21:27:07,784 DEBUG [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T21:27:07,784 DEBUG [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T21:27:07,784 INFO [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-17T21:27:07,803 DEBUG [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/.tmp/info/c953f77cd5e14905a3d87e6541b27485 is 95, key is TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b./info:/1731878674297/DeleteFamily/seqid=0 2024-11-17T21:27:07,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742461_1637 (size=6514) 2024-11-17T21:27:07,838 INFO [regionserver/a313eea8709e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T21:27:07,984 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T21:27:08,077 INFO [regionserver/a313eea8709e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-17T21:27:08,077 INFO [regionserver/a313eea8709e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-17T21:27:08,184 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T21:27:08,207 INFO [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/.tmp/info/c953f77cd5e14905a3d87e6541b27485 2024-11-17T21:27:08,224 DEBUG [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/.tmp/ns/91c6c1f1ee5f4d5f922bdffe3ed6dc0b is 93, key is TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b./ns:/1731878674254/DeleteFamily/seqid=0 2024-11-17T21:27:08,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742462_1638 (size=5906) 2024-11-17T21:27:08,384 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T21:27:08,585 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T21:27:08,627 INFO [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/.tmp/ns/91c6c1f1ee5f4d5f922bdffe3ed6dc0b 2024-11-17T21:27:08,644 DEBUG [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/.tmp/rep_barrier/18a4677249954a4c8fb611202c9e0558 is 102, key is TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b./rep_barrier:/1731878674254/DeleteFamily/seqid=0 2024-11-17T21:27:08,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742463_1639 (size=6025) 2024-11-17T21:27:08,785 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T21:27:08,985 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T21:27:09,048 INFO [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/.tmp/rep_barrier/18a4677249954a4c8fb611202c9e0558 2024-11-17T21:27:09,063 DEBUG [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/.tmp/table/4a61df624e3241ba957dbe94fe219ce0 is 96, key is TestAcidGuarantees,,1731878647591.8417a211937373f3c01713928c5cb18b./table:/1731878674254/DeleteFamily/seqid=0 2024-11-17T21:27:09,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742464_1640 (size=5936) 2024-11-17T21:27:09,185 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T21:27:09,385 DEBUG [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T21:27:09,467 INFO [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/.tmp/table/4a61df624e3241ba957dbe94fe219ce0 2024-11-17T21:27:09,471 DEBUG [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/.tmp/info/c953f77cd5e14905a3d87e6541b27485 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/info/c953f77cd5e14905a3d87e6541b27485 2024-11-17T21:27:09,473 INFO [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/info/c953f77cd5e14905a3d87e6541b27485, entries=12, sequenceid=90, filesize=6.4 K 2024-11-17T21:27:09,474 DEBUG [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/.tmp/ns/91c6c1f1ee5f4d5f922bdffe3ed6dc0b as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/ns/91c6c1f1ee5f4d5f922bdffe3ed6dc0b 2024-11-17T21:27:09,477 INFO [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/ns/91c6c1f1ee5f4d5f922bdffe3ed6dc0b, entries=8, sequenceid=90, filesize=5.8 K 2024-11-17T21:27:09,477 DEBUG [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/.tmp/rep_barrier/18a4677249954a4c8fb611202c9e0558 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/rep_barrier/18a4677249954a4c8fb611202c9e0558 2024-11-17T21:27:09,479 INFO [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/rep_barrier/18a4677249954a4c8fb611202c9e0558, entries=6, sequenceid=90, filesize=5.9 K 2024-11-17T21:27:09,480 DEBUG [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/.tmp/table/4a61df624e3241ba957dbe94fe219ce0 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/table/4a61df624e3241ba957dbe94fe219ce0 2024-11-17T21:27:09,482 INFO [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/table/4a61df624e3241ba957dbe94fe219ce0, entries=7, sequenceid=90, filesize=5.8 K 2024-11-17T21:27:09,483 INFO [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1699ms, sequenceid=90, compaction requested=false 2024-11-17T21:27:09,486 DEBUG [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-17T21:27:09,486 DEBUG [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T21:27:09,486 INFO [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T21:27:09,486 DEBUG [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731878827784Running coprocessor pre-close hooks at 1731878827784Disabling compacts and flushes for region at 1731878827784Disabling writes for close at 1731878827784Obtaining lock to block concurrent updates at 1731878827784Preparing flush snapshotting stores in 1588230740 at 1731878827784Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731878827784Flushing stores of hbase:meta,,1.1588230740 at 1731878827785 (+1 ms)Flushing 1588230740/info: creating writer at 1731878827785Flushing 1588230740/info: appending metadata at 1731878827803 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731878827803Flushing 1588230740/ns: creating writer at 1731878828210 (+407 ms)Flushing 1588230740/ns: appending metadata at 1731878828223 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731878828223Flushing 1588230740/rep_barrier: creating writer at 1731878828631 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731878828644 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731878828644Flushing 1588230740/table: creating writer at 1731878829050 (+406 ms)Flushing 1588230740/table: appending metadata at 1731878829063 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731878829063Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e40f91: reopening flushed file at 1731878829470 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76744723: reopening flushed file at 1731878829474 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@511a602f: reopening flushed file at 1731878829477 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18737891: reopening flushed file at 1731878829480 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1699ms, sequenceid=90, compaction requested=false at 1731878829483 (+3 ms)Writing region close event to WAL at 1731878829484 (+1 ms)Running coprocessor post-close hooks at 1731878829486 (+2 ms)Closed at 1731878829486 2024-11-17T21:27:09,487 DEBUG [RS_CLOSE_META-regionserver/a313eea8709e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T21:27:09,586 INFO [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(976): stopping server a313eea8709e,46445,1731878642828; all regions closed. 2024-11-17T21:27:09,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741834_1010 (size=25344) 2024-11-17T21:27:09,592 DEBUG [RS:0;a313eea8709e:46445 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/oldWALs 2024-11-17T21:27:09,592 INFO [RS:0;a313eea8709e:46445 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a313eea8709e%2C46445%2C1731878642828.meta:.meta(num 1731878647134) 2024-11-17T21:27:09,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741832_1008 (size=14473077) 2024-11-17T21:27:09,596 DEBUG [RS:0;a313eea8709e:46445 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/oldWALs 2024-11-17T21:27:09,596 INFO [RS:0;a313eea8709e:46445 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a313eea8709e%2C46445%2C1731878642828:(num 1731878646163) 2024-11-17T21:27:09,596 DEBUG [RS:0;a313eea8709e:46445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T21:27:09,596 INFO [RS:0;a313eea8709e:46445 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T21:27:09,596 INFO [RS:0;a313eea8709e:46445 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T21:27:09,596 INFO [RS:0;a313eea8709e:46445 {}] hbase.ChoreService(370): Chore service for: regionserver/a313eea8709e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-17T21:27:09,596 INFO [RS:0;a313eea8709e:46445 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T21:27:09,596 INFO [regionserver/a313eea8709e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T21:27:09,597 INFO [RS:0;a313eea8709e:46445 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46445 2024-11-17T21:27:09,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a313eea8709e,46445,1731878642828 2024-11-17T21:27:09,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T21:27:09,662 INFO [RS:0;a313eea8709e:46445 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T21:27:09,675 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a313eea8709e,46445,1731878642828] 2024-11-17T21:27:09,685 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a313eea8709e,46445,1731878642828 already deleted, retry=false 2024-11-17T21:27:09,685 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a313eea8709e,46445,1731878642828 expired; onlineServers=0 2024-11-17T21:27:09,686 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a313eea8709e,43623,1731878641945' ***** 2024-11-17T21:27:09,686 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-17T21:27:09,686 INFO [M:0;a313eea8709e:43623 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T21:27:09,686 INFO [M:0;a313eea8709e:43623 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T21:27:09,686 DEBUG [M:0;a313eea8709e:43623 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-17T21:27:09,686 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-17T21:27:09,686 DEBUG [M:0;a313eea8709e:43623 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-17T21:27:09,686 DEBUG [master/a313eea8709e:0:becomeActiveMaster-HFileCleaner.small.0-1731878645779 {}] cleaner.HFileCleaner(306): Exit Thread[master/a313eea8709e:0:becomeActiveMaster-HFileCleaner.small.0-1731878645779,5,FailOnTimeoutGroup] 2024-11-17T21:27:09,686 DEBUG [master/a313eea8709e:0:becomeActiveMaster-HFileCleaner.large.0-1731878645778 {}] cleaner.HFileCleaner(306): Exit Thread[master/a313eea8709e:0:becomeActiveMaster-HFileCleaner.large.0-1731878645778,5,FailOnTimeoutGroup] 2024-11-17T21:27:09,686 INFO [M:0;a313eea8709e:43623 {}] hbase.ChoreService(370): Chore service for: master/a313eea8709e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-17T21:27:09,686 INFO [M:0;a313eea8709e:43623 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T21:27:09,686 DEBUG [M:0;a313eea8709e:43623 {}] master.HMaster(1795): Stopping service threads 2024-11-17T21:27:09,686 INFO [M:0;a313eea8709e:43623 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-17T21:27:09,687 INFO [M:0;a313eea8709e:43623 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T21:27:09,687 ERROR [M:0;a313eea8709e:43623 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-17T21:27:09,687 INFO [M:0;a313eea8709e:43623 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T21:27:09,687 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-17T21:27:09,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-17T21:27:09,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T21:27:09,696 DEBUG [M:0;a313eea8709e:43623 {}] zookeeper.ZKUtil(347): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-17T21:27:09,696 WARN [M:0;a313eea8709e:43623 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-17T21:27:09,696 INFO [M:0;a313eea8709e:43623 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/.lastflushedseqids 2024-11-17T21:27:09,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742465_1641 (size=72) 2024-11-17T21:27:09,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T21:27:09,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46445-0x1014aaed9aa0001, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T21:27:09,776 INFO [RS:0;a313eea8709e:46445 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T21:27:09,776 INFO [RS:0;a313eea8709e:46445 {}] regionserver.HRegionServer(1031): Exiting; stopping=a313eea8709e,46445,1731878642828; zookeeper connection closed. 2024-11-17T21:27:09,776 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@715ac169 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@715ac169 2024-11-17T21:27:09,777 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-17T21:27:10,105 INFO [M:0;a313eea8709e:43623 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T21:27:10,105 INFO [M:0;a313eea8709e:43623 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-17T21:27:10,105 DEBUG [M:0;a313eea8709e:43623 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T21:27:10,105 INFO [M:0;a313eea8709e:43623 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T21:27:10,105 DEBUG [M:0;a313eea8709e:43623 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T21:27:10,105 DEBUG [M:0;a313eea8709e:43623 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T21:27:10,105 DEBUG [M:0;a313eea8709e:43623 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T21:27:10,105 INFO [M:0;a313eea8709e:43623 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=755.39 KB heapSize=928.80 KB 2024-11-17T21:27:10,121 DEBUG [M:0;a313eea8709e:43623 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d8c8e1b1d4a646e4a015a18d1d3e874d is 82, key is hbase:meta,,1/info:regioninfo/1731878647213/Put/seqid=0 2024-11-17T21:27:10,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742466_1642 (size=5672) 2024-11-17T21:27:10,525 INFO [M:0;a313eea8709e:43623 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2140 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d8c8e1b1d4a646e4a015a18d1d3e874d 2024-11-17T21:27:10,545 DEBUG [M:0;a313eea8709e:43623 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/664e83b67da64f3da4ffff87b280f7d1 is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1731878796953/Put/seqid=0 2024-11-17T21:27:10,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742467_1643 (size=44121) 2024-11-17T21:27:10,949 INFO [M:0;a313eea8709e:43623 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=754.83 KB at sequenceid=2140 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/664e83b67da64f3da4ffff87b280f7d1 2024-11-17T21:27:10,952 INFO [M:0;a313eea8709e:43623 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 664e83b67da64f3da4ffff87b280f7d1 2024-11-17T21:27:10,965 DEBUG [M:0;a313eea8709e:43623 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f238e9b639724612a27047701dfe8101 is 69, key is a313eea8709e,46445,1731878642828/rs:state/1731878645914/Put/seqid=0 2024-11-17T21:27:10,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073742468_1644 (size=5156) 2024-11-17T21:27:11,368 INFO [M:0;a313eea8709e:43623 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2140 (bloomFilter=true), to=hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f238e9b639724612a27047701dfe8101 2024-11-17T21:27:11,372 DEBUG [M:0;a313eea8709e:43623 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d8c8e1b1d4a646e4a015a18d1d3e874d as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d8c8e1b1d4a646e4a015a18d1d3e874d 2024-11-17T21:27:11,374 INFO [M:0;a313eea8709e:43623 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d8c8e1b1d4a646e4a015a18d1d3e874d, entries=8, sequenceid=2140, filesize=5.5 K 2024-11-17T21:27:11,375 DEBUG [M:0;a313eea8709e:43623 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/664e83b67da64f3da4ffff87b280f7d1 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/664e83b67da64f3da4ffff87b280f7d1 2024-11-17T21:27:11,377 INFO [M:0;a313eea8709e:43623 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 664e83b67da64f3da4ffff87b280f7d1 2024-11-17T21:27:11,377 INFO [M:0;a313eea8709e:43623 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/664e83b67da64f3da4ffff87b280f7d1, entries=170, sequenceid=2140, filesize=43.1 K 2024-11-17T21:27:11,378 DEBUG [M:0;a313eea8709e:43623 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f238e9b639724612a27047701dfe8101 as hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f238e9b639724612a27047701dfe8101 2024-11-17T21:27:11,380 INFO [M:0;a313eea8709e:43623 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45699/user/jenkins/test-data/c165af93-33b9-be43-74f2-c29d153b7249/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f238e9b639724612a27047701dfe8101, entries=1, sequenceid=2140, filesize=5.0 K 2024-11-17T21:27:11,381 INFO [M:0;a313eea8709e:43623 {}] regionserver.HRegion(3140): Finished flush of dataSize ~755.39 KB/773516, heapSize ~928.50 KB/950784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2140, compaction requested=false 2024-11-17T21:27:11,382 INFO [M:0;a313eea8709e:43623 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T21:27:11,382 DEBUG [M:0;a313eea8709e:43623 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731878830105Disabling compacts and flushes for region at 1731878830105Disabling writes for close at 1731878830105Obtaining lock to block concurrent updates at 1731878830105Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731878830105Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=773516, getHeapSize=951024, getOffHeapSize=0, getCellsCount=2335 at 1731878830106 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731878830106Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731878830106Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731878830121 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731878830121Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731878830528 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731878830545 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731878830545Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731878830952 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731878830964 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731878830965 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b866dd: reopening flushed file at 1731878831371 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a9ae1eb: reopening flushed file at 1731878831374 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cd37a35: reopening flushed file at 1731878831377 (+3 ms)Finished flush of dataSize ~755.39 KB/773516, heapSize ~928.50 KB/950784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2140, compaction requested=false at 1731878831381 (+4 ms)Writing region close event to WAL at 1731878831382 (+1 ms)Closed at 1731878831382 2024-11-17T21:27:11,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35609 is added to blk_1073741830_1006 (size=913815) 2024-11-17T21:27:11,387 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T21:27:11,387 INFO [M:0;a313eea8709e:43623 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-17T21:27:11,387 INFO [M:0;a313eea8709e:43623 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43623 2024-11-17T21:27:11,387 INFO [M:0;a313eea8709e:43623 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T21:27:11,554 INFO [M:0;a313eea8709e:43623 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T21:27:11,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T21:27:11,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43623-0x1014aaed9aa0000, quorum=127.0.0.1:62975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T21:27:11,559 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@42a79998{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T21:27:11,561 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T21:27:11,561 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T21:27:11,561 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T21:27:11,562 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/hadoop.log.dir/,STOPPED} 2024-11-17T21:27:11,564 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T21:27:11,564 WARN [BP-2092338727-172.17.0.2-1731878638249 heartbeating to localhost/127.0.0.1:45699 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T21:27:11,564 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T21:27:11,564 WARN [BP-2092338727-172.17.0.2-1731878638249 heartbeating to localhost/127.0.0.1:45699 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2092338727-172.17.0.2-1731878638249 (Datanode Uuid f9e820aa-39dd-40f9-8116-567c831aafd3) service to localhost/127.0.0.1:45699 2024-11-17T21:27:11,567 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/cluster_6acd51c4-a1e3-75e6-757b-0efaa889834a/data/data1/current/BP-2092338727-172.17.0.2-1731878638249 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T21:27:11,567 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/cluster_6acd51c4-a1e3-75e6-757b-0efaa889834a/data/data2/current/BP-2092338727-172.17.0.2-1731878638249 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T21:27:11,568 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T21:27:11,575 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@322e478f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T21:27:11,576 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T21:27:11,576 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T21:27:11,576 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T21:27:11,576 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9801a5ea-53d9-1009-722e-cfa793c6cb76/hadoop.log.dir/,STOPPED} 2024-11-17T21:27:11,591 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-17T21:27:11,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down